From 32621be708cede8ac11519164c88c900688b2019 Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Sun, 16 Mar 2014 11:23:16 +0100 Subject: [PATCH 001/381] move gl elements to ext subdirectory --- ext/gl/gstglmosaic.c | 365 +++++++++++++++++++++++++++++++++++++++ ext/gl/gstglmosaic.h | 55 ++++++ ext/gl/gstglvideomixer.c | 294 +++++++++++++++++++++++++++++++ ext/gl/gstglvideomixer.h | 55 ++++++ 4 files changed, 769 insertions(+) create mode 100644 ext/gl/gstglmosaic.c create mode 100644 ext/gl/gstglmosaic.h create mode 100644 ext/gl/gstglvideomixer.c create mode 100644 ext/gl/gstglvideomixer.h diff --git a/ext/gl/gstglmosaic.c b/ext/gl/gstglmosaic.c new file mode 100644 index 0000000000..040b88c5ac --- /dev/null +++ b/ext/gl/gstglmosaic.c @@ -0,0 +1,365 @@ +/* + * GStreamer + * Copyright (C) 2009 Julien Isorce + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +/** + * SECTION:element-glmosaic + * + * glmixer sub element. N gl sink pads to 1 source pad. + * N + 1 OpenGL contexts shared together. + * N <= 6 because the rendering is more a like a cube than a mosaic + * Each opengl input stream is rendered on a cube face + * + * + * Examples + * |[ + * gst-launch-0.10 videotestsrc ! "video/x-raw-yuv, format=(fourcc)YUY2" ! glupload ! queue ! glmosaic name=m ! glimagesink videotestsrc pattern=12 ! "video/x-raw-yuv, format=(fourcc)I420, framerate=(fraction)5/1, width=100, height=200" ! glupload ! queue ! m. videotestsrc ! "video/x-raw-rgb, framerate=(fraction)15/1, width=1500, height=1500" ! glupload ! gleffects effect=3 ! queue ! m. videotestsrc ! glupload ! gleffects effect=2 ! queue ! m. videotestsrc ! glupload ! glfiltercube ! queue ! m. videotestsrc ! glupload ! gleffects effect=6 ! queue ! m. + * ]| + * FBO (Frame Buffer Object) is required. + * + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "gstglmosaic.h" + +#define GST_CAT_DEFAULT gst_gl_mosaic_debug +GST_DEBUG_CATEGORY_STATIC (GST_CAT_DEFAULT); + +enum +{ + PROP_0, +}; + +#define DEBUG_INIT \ + GST_DEBUG_CATEGORY_INIT (gst_gl_mosaic_debug, "glmosaic", 0, "glmosaic element"); + +G_DEFINE_TYPE_WITH_CODE (GstGLMosaic, gst_gl_mosaic, GST_TYPE_GL_MIXER, + DEBUG_INIT); + +static void gst_gl_mosaic_set_property (GObject * object, guint prop_id, + const GValue * value, GParamSpec * pspec); +static void gst_gl_mosaic_get_property (GObject * object, guint prop_id, + GValue * value, GParamSpec * pspec); + +static void gst_gl_mosaic_reset (GstGLMixer * mixer); +static gboolean gst_gl_mosaic_init_shader (GstGLMixer * mixer, + GstCaps * outcaps); + +static gboolean gst_gl_mosaic_process_textures (GstGLMixer * mixer, + GPtrArray * frames, guint out_tex); +static void gst_gl_mosaic_callback (gpointer stuff); + +//vertex source +static const gchar *mosaic_v_src = + "uniform mat4 u_matrix; \n" + "uniform float xrot_degree, yrot_degree, zrot_degree; \n" + "attribute vec4 a_position; \n" + "attribute vec2 a_texCoord; \n" + "varying vec2 v_texCoord; \n" + "void main() \n" + "{ \n" + " float PI = 3.14159265; \n" + " float xrot = xrot_degree*2.0*PI/360.0; \n" + " float yrot = yrot_degree*2.0*PI/360.0; \n" + " float zrot = zrot_degree*2.0*PI/360.0; \n" + " mat4 matX = mat4 ( \n" + " 1.0, 0.0, 0.0, 0.0, \n" + " 0.0, cos(xrot), sin(xrot), 0.0, \n" + " 0.0, -sin(xrot), cos(xrot), 0.0, \n" + " 0.0, 0.0, 0.0, 1.0 ); \n" + " mat4 matY = mat4 ( \n" + " cos(yrot), 0.0, -sin(yrot), 0.0, \n" + " 0.0, 1.0, 0.0, 0.0, \n" + " sin(yrot), 0.0, cos(yrot), 0.0, \n" + " 0.0, 0.0, 0.0, 1.0 ); \n" + " mat4 matZ = mat4 ( \n" + " cos(zrot), sin(zrot), 0.0, 0.0, \n" + " -sin(zrot), cos(zrot), 0.0, 0.0, \n" + " 0.0, 0.0, 1.0, 0.0, \n" + " 0.0, 0.0, 0.0, 1.0 ); \n" + " gl_Position = u_matrix * matZ * matY * matX * a_position; \n" + " v_texCoord = a_texCoord; \n" + "} \n"; + +//fragment source +static const gchar *mosaic_f_src = + "uniform sampler2D s_texture; \n" + "varying vec2 v_texCoord; \n" + "void main() \n" + "{ \n" + //" gl_FragColor = vec4( 1.0, 0.5, 1.0, 1.0 );\n" + " gl_FragColor = texture2D( s_texture, v_texCoord );\n" + "} \n"; + +static void +gst_gl_mosaic_class_init (GstGLMosaicClass * klass) +{ + GObjectClass *gobject_class; + GstElementClass *element_class; + + gobject_class = (GObjectClass *) klass; + element_class = GST_ELEMENT_CLASS (klass); + + gobject_class->set_property = gst_gl_mosaic_set_property; + gobject_class->get_property = gst_gl_mosaic_get_property; + + gst_element_class_set_metadata (element_class, "OpenGL mosaic", + "Filter/Effect/Video", "OpenGL mosaic", + "Julien Isorce "); + + GST_GL_MIXER_CLASS (klass)->set_caps = gst_gl_mosaic_init_shader; + GST_GL_MIXER_CLASS (klass)->reset = gst_gl_mosaic_reset; + GST_GL_MIXER_CLASS (klass)->process_textures = gst_gl_mosaic_process_textures; +} + +static void +gst_gl_mosaic_init (GstGLMosaic * mosaic) +{ + mosaic->shader = NULL; + mosaic->input_frames = NULL; +} + +static void +gst_gl_mosaic_set_property (GObject * object, guint prop_id, + const GValue * value, GParamSpec * pspec) +{ + //GstGLMosaic *mixer = GST_GL_MOSAIC (object); + + switch (prop_id) { + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static void +gst_gl_mosaic_get_property (GObject * object, guint prop_id, + GValue * value, GParamSpec * pspec) +{ + //GstGLMosaic* mixer = GST_GL_MOSAIC (object); + + switch (prop_id) { + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static void +gst_gl_mosaic_reset (GstGLMixer * mixer) +{ + GstGLMosaic *mosaic = GST_GL_MOSAIC (mixer); + + mosaic->input_frames = NULL; + + //blocking call, wait the opengl thread has destroyed the shader + if (mosaic->shader) + gst_gl_context_del_shader (mixer->context, mosaic->shader); + mosaic->shader = NULL; +} + +static gboolean +gst_gl_mosaic_init_shader (GstGLMixer * mixer, GstCaps * outcaps) +{ + GstGLMosaic *mosaic = GST_GL_MOSAIC (mixer); + + //blocking call, wait the opengl thread has compiled the shader + return gst_gl_context_gen_shader (mixer->context, mosaic_v_src, mosaic_f_src, + &mosaic->shader); +} + +static gboolean +gst_gl_mosaic_process_textures (GstGLMixer * mix, GPtrArray * frames, + guint out_tex) +{ + GstGLMosaic *mosaic = GST_GL_MOSAIC (mix); + + mosaic->input_frames = frames; + + //blocking call, use a FBO + gst_gl_context_use_fbo_v2 (mix->context, + GST_VIDEO_INFO_WIDTH (&mix->out_info), + GST_VIDEO_INFO_HEIGHT (&mix->out_info), mix->fbo, mix->depthbuffer, + out_tex, gst_gl_mosaic_callback, (gpointer) mosaic); + + return TRUE; +} + +/* opengl scene, params: input texture (not the output mixer->texture) */ +static void +gst_gl_mosaic_callback (gpointer stuff) +{ + GstGLMosaic *mosaic = GST_GL_MOSAIC (stuff); + GstGLMixer *mixer = GST_GL_MIXER (mosaic); + GstGLFuncs *gl = mixer->context->gl_vtable; + + static GLfloat xrot = 0; + static GLfloat yrot = 0; + static GLfloat zrot = 0; + + GLint attr_position_loc = 0; + GLint attr_texture_loc = 0; + + const GLfloat matrix[] = { + 0.5f, 0.0f, 0.0f, 0.0f, + 0.0f, 0.5f, 0.0f, 0.0f, + 0.0f, 0.0f, 0.5f, 0.0f, + 0.0f, 0.0f, 0.0f, 1.0f + }; + const GLushort indices[] = { + 0, 1, 2, + 0, 2, 3 + }; + + guint count = 0; + + gst_gl_context_clear_shader (mixer->context); + gl->BindTexture (GL_TEXTURE_2D, 0); + gl->Disable (GL_TEXTURE_2D); + + gl->Enable (GL_DEPTH_TEST); + + gl->ClearColor (0.0, 0.0, 0.0, 0.0); + gl->Clear (GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT); + + gst_gl_shader_use (mosaic->shader); + + attr_position_loc = + gst_gl_shader_get_attribute_location (mosaic->shader, "a_position"); + attr_texture_loc = + gst_gl_shader_get_attribute_location (mosaic->shader, "a_texCoord"); + + while (count < mosaic->input_frames->len && count < 6) { + GstGLMixerFrameData *frame; + GLfloat *v_vertices; + guint in_tex; + guint width, height; + + frame = g_ptr_array_index (mosaic->input_frames, count); + in_tex = frame->texture; + width = GST_VIDEO_INFO_WIDTH (&frame->pad->in_info); + height = GST_VIDEO_INFO_HEIGHT (&frame->pad->in_info); + + if (!frame || !in_tex || width <= 0 || height <= 0) { + GST_DEBUG ("skipping texture:%u frame:%p width:%u height %u", + in_tex, frame, width, height); + count++; + continue; + } + + GST_TRACE ("processing texture:%u dimensions:%ux%u", in_tex, width, height); + + /* *INDENT-OFF* */ + v_vertices = (GLfloat[]) { + /* front face */ + 1.0f, 1.0f, -1.0f, + 1.0f, 0.0f, + 1.0f, -1.0f, -1.0f, + 1.0f, 1.0f, + -1.0f, -1.0f, -1.0f, + 0.0f, 1.0f, + -1.0f, 1.0f, -1.0f, + 0.0f, 0.0f, + /* right face */ + 1.0f, 1.0f, 1.0f, + 1.0f, 0.0f, + 1.0f, -1.0f, 1.0f, + 0.0f, 0.0f, + 1.0f, -1.0f, -1.0f, + 0.0f, 1.0f, + 1.0f, 1.0f, -1.0f, + 1.0f, 1.0f, + /* left face */ + -1.0f, 1.0f, 1.0f, + 1.0f, 0.0f, + -1.0f, 1.0f, -1.0f, + 1.0f, 1.0f, + -1.0f, -1.0f, -1.0f, + 0.0f, 1.0f, + -1.0f, -1.0f, 1.0f, + 0.0f, 0.0f, + /* top face */ + 1.0f, -1.0f, 1.0f, + 1.0f, 0.0f, + -1.0f, -1.0f, 1.0f, + 0.0f, 0.0f, + -1.0f, -1.0f, -1.0f, + 0.0f, 1.0f, + 1.0f, -1.0f, -1.0f, + 1.0f, 1.0f, + /* bottom face */ + 1.0f, 1.0f, 1.0f, + 1.0f, 0.0f, + 1.0f, 1.0f, -1.0f, + 1.0f, 1.0f, + -1.0f, 1.0f, -1.0f, + 0.0f, 1.0f, + -1.0f, 1.0f, 1.0f, + 0.0f, 0.0f, + /* back face */ + 1.0f, 1.0f, 1.0f, + 1.0f, 0.0f, + -1.0f, 1.0f, 1.0f, + 0.0f, 0.0f, + -1.0f, -1.0f, 1.0f, + 0.0f, 1.0f, + 1.0f, -1.0f, 1.0f, + 1.0f, 1.0f + }; + /* *INDENT-ON* */ + + gl->VertexAttribPointer (attr_position_loc, 3, GL_FLOAT, + GL_FALSE, 5 * sizeof (GLfloat), &v_vertices[5 * 4 * count]); + + gl->VertexAttribPointer (attr_texture_loc, 2, GL_FLOAT, + GL_FALSE, 5 * sizeof (GLfloat), &v_vertices[5 * 4 * count + 3]); + + gl->EnableVertexAttribArray (attr_position_loc); + gl->EnableVertexAttribArray (attr_texture_loc); + + gl->ActiveTexture (GL_TEXTURE0); + gl->BindTexture (GL_TEXTURE_2D, in_tex); + gst_gl_shader_set_uniform_1i (mosaic->shader, "s_texture", 0); + gst_gl_shader_set_uniform_1f (mosaic->shader, "xrot_degree", xrot); + gst_gl_shader_set_uniform_1f (mosaic->shader, "yrot_degree", yrot); + gst_gl_shader_set_uniform_1f (mosaic->shader, "zrot_degree", zrot); + gst_gl_shader_set_uniform_matrix_4fv (mosaic->shader, "u_matrix", 1, + GL_FALSE, matrix); + + gl->DrawElements (GL_TRIANGLES, 6, GL_UNSIGNED_SHORT, indices); + + ++count; + } + + gl->DisableVertexAttribArray (attr_position_loc); + gl->DisableVertexAttribArray (attr_texture_loc); + + gl->BindTexture (GL_TEXTURE_2D, 0); + + gl->Disable (GL_DEPTH_TEST); + + gst_gl_context_clear_shader (mixer->context); + + xrot += 0.6f; + yrot += 0.4f; + zrot += 0.8f; +} diff --git a/ext/gl/gstglmosaic.h b/ext/gl/gstglmosaic.h new file mode 100644 index 0000000000..1da9dbe29c --- /dev/null +++ b/ext/gl/gstglmosaic.h @@ -0,0 +1,55 @@ +/* + * GStreamer + * Copyright (C) 2009 Julien Isorce + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifndef _GST_GL_MOSAIC_H_ +#define _GST_GL_MOSAIC_H_ + +#include + +G_BEGIN_DECLS + +#define GST_TYPE_GL_MOSAIC (gst_gl_mosaic_get_type()) +#define GST_GL_MOSAIC(obj) (G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_GL_MOSAIC,GstGLMosaic)) +#define GST_IS_GL_MOSAIC(obj) (G_TYPE_CHECK_INSTANCE_TYPE((obj),GST_TYPE_GL_MOSAIC)) +#define GST_GL_MOSAIC_CLASS(klass) (G_TYPE_CHECK_CLASS_CAST((klass) ,GST_TYPE_GL_MOSAIC,GstGLMosaicClass)) +#define GST_IS_GL_MOSAIC_CLASS(klass) (G_TYPE_CHECK_CLASS_TYPE((klass) ,GST_TYPE_GL_MOSAIC)) +#define GST_GL_MOSAIC_GET_CLASS(obj) (G_TYPE_INSTANCE_GET_CLASS((obj) ,GST_TYPE_GL_MOSAIC,GstGLMosaicClass)) + +typedef struct _GstGLMosaic GstGLMosaic; +typedef struct _GstGLMosaicClass GstGLMosaicClass; + +struct _GstGLMosaic +{ + GstGLMixer mixer; + + GstGLShader *shader; + GPtrArray *input_frames; +}; + +struct _GstGLMosaicClass +{ + GstGLMixerClass mixer_class; +}; + +GType gst_gl_mosaic_get_type (void); + +G_END_DECLS + +#endif /* _GST_GLFILTERCUBE_H_ */ diff --git a/ext/gl/gstglvideomixer.c b/ext/gl/gstglvideomixer.c new file mode 100644 index 0000000000..d72f83ba7a --- /dev/null +++ b/ext/gl/gstglvideomixer.c @@ -0,0 +1,294 @@ +/* + * GStreamer + * Copyright (C) 2009 Julien Isorce + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +/** + * SECTION:element-glvideomixer + * + * glmixer sub element. N gl sink pads to 1 source pad. + * N + 1 OpenGL contexts shared together. + * N <= 6 because the rendering is more a like a cube than a video_mixer + * Each opengl input stream is rendered on a cube face + * + * + * Examples + * |[ + * gst-launch-0.10 videotestsrc ! "video/x-raw-yuv, format=(fourcc)YUY2" ! glupload ! queue ! glvideomixer name=m ! glimagesink videotestsrc pattern=12 ! "video/x-raw-yuv, format=(fourcc)I420, framerate=(fraction)5/1, width=100, height=200" ! glupload ! queue ! m. videotestsrc ! "video/x-raw-rgb, framerate=(fraction)15/1, width=1500, height=1500" ! glupload ! gleffects effect=3 ! queue ! m. videotestsrc ! glupload ! gleffects effect=2 ! queue ! m. videotestsrc ! glupload ! glfiltercube ! queue ! m. videotestsrc ! glupload ! gleffects effect=6 ! queue ! m. + * ]| + * FBO (Frame Buffer Object) is required. + * + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "gstglvideomixer.h" + +#define GST_CAT_DEFAULT gst_gl_video_mixer_debug +GST_DEBUG_CATEGORY_STATIC (GST_CAT_DEFAULT); + +enum +{ + PROP_0, +}; + +#define DEBUG_INIT \ + GST_DEBUG_CATEGORY_INIT (gst_gl_video_mixer_debug, "glvideomixer", 0, "glvideomixer element"); + +G_DEFINE_TYPE_WITH_CODE (GstGLVideoMixer, gst_gl_video_mixer, GST_TYPE_GL_MIXER, + DEBUG_INIT); + +static void gst_gl_video_mixer_set_property (GObject * object, guint prop_id, + const GValue * value, GParamSpec * pspec); +static void gst_gl_video_mixer_get_property (GObject * object, guint prop_id, + GValue * value, GParamSpec * pspec); + +static void gst_gl_video_mixer_reset (GstGLMixer * mixer); +static gboolean gst_gl_video_mixer_init_shader (GstGLMixer * mixer, + GstCaps * outcaps); + +static gboolean gst_gl_video_mixer_process_textures (GstGLMixer * mixer, + GPtrArray * in_frames, guint out_tex); +static void gst_gl_video_mixer_callback (gpointer stuff); + +/* vertex source */ +static const gchar *video_mixer_v_src = + "attribute vec4 a_position; \n" + "attribute vec2 a_texCoord; \n" + "uniform float x_scale; \n" + "uniform float y_scale; \n" + "varying vec2 v_texCoord; \n" + "void main() \n" + "{ \n" + " gl_Position = a_position * vec4(x_scale, y_scale, 1.0, 1.0);\n" + " v_texCoord = a_texCoord; \n" "}"; + +/* fragment source */ +static const gchar *video_mixer_f_src = + "uniform sampler2D texture; \n" + "varying vec2 v_texCoord; \n" + "void main() \n" + "{ \n" + " vec4 rgba = texture2D( texture, v_texCoord );\n" + " gl_FragColor = vec4(rgba.rgb, 1.0);\n" + "} \n"; + +static void +gst_gl_video_mixer_class_init (GstGLVideoMixerClass * klass) +{ + GObjectClass *gobject_class; + GstElementClass *element_class; + + gobject_class = (GObjectClass *) klass; + element_class = GST_ELEMENT_CLASS (klass); + + gobject_class->set_property = gst_gl_video_mixer_set_property; + gobject_class->get_property = gst_gl_video_mixer_get_property; + + gst_element_class_set_metadata (element_class, "OpenGL video_mixer", + "Filter/Effect/Video", "OpenGL video_mixer", + "Julien Isorce "); + + GST_GL_MIXER_CLASS (klass)->set_caps = gst_gl_video_mixer_init_shader; + GST_GL_MIXER_CLASS (klass)->reset = gst_gl_video_mixer_reset; + GST_GL_MIXER_CLASS (klass)->process_textures = + gst_gl_video_mixer_process_textures; +} + +static void +gst_gl_video_mixer_init (GstGLVideoMixer * video_mixer) +{ + video_mixer->shader = NULL; + video_mixer->input_frames = NULL; +} + +static void +gst_gl_video_mixer_set_property (GObject * object, guint prop_id, + const GValue * value, GParamSpec * pspec) +{ + switch (prop_id) { + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static void +gst_gl_video_mixer_get_property (GObject * object, guint prop_id, + GValue * value, GParamSpec * pspec) +{ + switch (prop_id) { + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static void +gst_gl_video_mixer_reset (GstGLMixer * mixer) +{ + GstGLVideoMixer *video_mixer = GST_GL_VIDEO_MIXER (mixer); + + video_mixer->input_frames = NULL; + + if (video_mixer->shader) + gst_gl_context_del_shader (mixer->context, video_mixer->shader); + video_mixer->shader = NULL; +} + +static gboolean +gst_gl_video_mixer_init_shader (GstGLMixer * mixer, GstCaps * outcaps) +{ + GstGLVideoMixer *video_mixer = GST_GL_VIDEO_MIXER (mixer); + + return gst_gl_context_gen_shader (mixer->context, video_mixer_v_src, + video_mixer_f_src, &video_mixer->shader); +} + +static gboolean +gst_gl_video_mixer_process_textures (GstGLMixer * mix, GPtrArray * frames, + guint out_tex) +{ + GstGLVideoMixer *video_mixer = GST_GL_VIDEO_MIXER (mix); + + video_mixer->input_frames = frames; + + gst_gl_context_use_fbo_v2 (mix->context, + GST_VIDEO_INFO_WIDTH (&mix->out_info), + GST_VIDEO_INFO_HEIGHT (&mix->out_info), mix->fbo, mix->depthbuffer, + out_tex, gst_gl_video_mixer_callback, (gpointer) video_mixer); + + return TRUE; +} + +/* opengl scene, params: input texture (not the output mixer->texture) */ +static void +gst_gl_video_mixer_callback (gpointer stuff) +{ + GstGLVideoMixer *video_mixer = GST_GL_VIDEO_MIXER (stuff); + GstGLMixer *mixer = GST_GL_MIXER (video_mixer); + GstGLFuncs *gl = mixer->context->gl_vtable; + + GLint attr_position_loc = 0; + GLint attr_texture_loc = 0; + guint out_width, out_height; + + const GLushort indices[] = { + 0, 1, 2, + 0, 2, 3 + }; + + guint count = 0; + + out_width = GST_VIDEO_INFO_WIDTH (&mixer->out_info); + out_height = GST_VIDEO_INFO_HEIGHT (&mixer->out_info); + + gst_gl_context_clear_shader (mixer->context); + gl->BindTexture (GL_TEXTURE_2D, 0); + gl->Disable (GL_TEXTURE_2D); + + gl->Disable (GL_DEPTH_TEST); + gl->Disable (GL_CULL_FACE); + + gl->ClearColor (0.0, 0.0, 0.0, 0.0); + gl->Clear (GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT); + + gst_gl_shader_use (video_mixer->shader); + + attr_position_loc = + gst_gl_shader_get_attribute_location (video_mixer->shader, "a_position"); + attr_texture_loc = + gst_gl_shader_get_attribute_location (video_mixer->shader, "a_texCoord"); + + gl->Enable (GL_BLEND); + + while (count < video_mixer->input_frames->len) { + GstGLMixerFrameData *frame; + GLfloat *v_vertices; + guint in_tex; + guint in_width, in_height; + gfloat w, h; + + frame = g_ptr_array_index (video_mixer->input_frames, count); + in_tex = frame->texture; + in_width = GST_VIDEO_INFO_WIDTH (&frame->pad->in_info); + in_height = GST_VIDEO_INFO_HEIGHT (&frame->pad->in_info); + + if (!frame || !in_tex || in_width <= 0 || in_height <= 0) { + GST_DEBUG ("skipping texture:%u frame:%p width:%u height %u", + in_tex, frame, in_width, in_height); + count++; + continue; + } + + GST_TRACE ("processing texture:%u dimensions:%ux%u", in_tex, in_width, + in_height); + + w = ((gfloat) in_width / (gfloat) out_width); + h = ((gfloat) in_height / (gfloat) out_height); + GST_TRACE ("processing texture:%u dimensions:%ux%u, %fx%f", in_tex, + in_width, in_height, w, h); + + /* *INDENT-OFF* */ + v_vertices = (GLfloat[]) { + /* front face */ + -1.0, -1.0, -1.0f, + 0.0f, 0.0f, + 1.0, -1.0, -1.0f, + 1.0f, 0.0f, + 1.0, 1.0, -1.0f, + 1.0f, 1.0f, + -1.0, 1.0, -1.0f, + 0.0f, 1.0f, + }; + /* *INDENT-ON* */ + + gl->VertexAttribPointer (attr_position_loc, 3, GL_FLOAT, + GL_FALSE, 5 * sizeof (GLfloat), &v_vertices[0]); + + gl->VertexAttribPointer (attr_texture_loc, 2, GL_FLOAT, + GL_FALSE, 5 * sizeof (GLfloat), &v_vertices[3]); + + gl->EnableVertexAttribArray (attr_position_loc); + gl->EnableVertexAttribArray (attr_texture_loc); + + gl->BlendFunc (GL_SRC_COLOR, GL_ONE_MINUS_SRC_COLOR); + gl->BlendEquation (GL_FUNC_ADD); + + gl->ActiveTexture (GL_TEXTURE0); + gl->BindTexture (GL_TEXTURE_2D, in_tex); + gst_gl_shader_set_uniform_1i (video_mixer->shader, "texture", 0); + gst_gl_shader_set_uniform_1f (video_mixer->shader, "x_scale", w); + gst_gl_shader_set_uniform_1f (video_mixer->shader, "y_scale", h); + + gl->DrawElements (GL_TRIANGLES, 6, GL_UNSIGNED_SHORT, indices); + + ++count; + } + + gl->DisableVertexAttribArray (attr_position_loc); + gl->DisableVertexAttribArray (attr_texture_loc); + + gl->BindTexture (GL_TEXTURE_2D, 0); + + gl->Disable (GL_BLEND); + + gst_gl_context_clear_shader (mixer->context); +} diff --git a/ext/gl/gstglvideomixer.h b/ext/gl/gstglvideomixer.h new file mode 100644 index 0000000000..716c60a7e0 --- /dev/null +++ b/ext/gl/gstglvideomixer.h @@ -0,0 +1,55 @@ +/* + * GStreamer + * Copyright (C) 2009 Julien Isorce + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifndef _GST_GL_VIDEO_MIXER_H_ +#define _GST_GL_VIDEO_MIXER_H_ + +#include + +G_BEGIN_DECLS + +#define GST_TYPE_GL_VIDEO_MIXER (gst_gl_video_mixer_get_type()) +#define GST_GL_VIDEO_MIXER(obj) (G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_GL_VIDEO_MIXER,GstGLVideoMixer)) +#define GST_IS_GL_VIDEO_MIXER(obj) (G_TYPE_CHECK_INSTANCE_TYPE((obj),GST_TYPE_GL_VIDEO_MIXER)) +#define GST_GL_VIDEO_MIXER_CLASS(klass) (G_TYPE_CHECK_CLASS_CAST((klass) ,GST_TYPE_GL_VIDEO_MIXER,GstGLVideoMixerClass)) +#define GST_IS_GL_VIDEO_MIXER_CLASS(klass) (G_TYPE_CHECK_CLASS_TYPE((klass) ,GST_TYPE_GL_VIDEO_MIXER)) +#define GST_GL_VIDEO_MIXER_GET_CLASS(obj) (G_TYPE_INSTANCE_GET_CLASS((obj) ,GST_TYPE_GL_VIDEO_MIXER,GstGLVideoMixerClass)) + +typedef struct _GstGLVideoMixer GstGLVideoMixer; +typedef struct _GstGLVideoMixerClass GstGLVideoMixerClass; + +struct _GstGLVideoMixer +{ + GstGLMixer mixer; + + GstGLShader *shader; + GPtrArray *input_frames; +}; + +struct _GstGLVideoMixerClass +{ + GstGLMixerClass mixer_class; +}; + +GType gst_gl_video_mixer_get_type (void); + +G_END_DECLS + +#endif /* _GST_GLFILTERCUBE_H_ */ From 665cc2ba21131430fdf86fe9c3847b41181acd47 Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Wed, 2 Apr 2014 22:43:41 +1100 Subject: [PATCH 002/381] gl: fix assignment of temporary variables --- ext/gl/gstglmosaic.c | 94 +++++++++++++++------------------------- ext/gl/gstglvideomixer.c | 28 +++++------- 2 files changed, 45 insertions(+), 77 deletions(-) diff --git a/ext/gl/gstglmosaic.c b/ext/gl/gstglmosaic.c index 040b88c5ac..d5ff754199 100644 --- a/ext/gl/gstglmosaic.c +++ b/ext/gl/gstglmosaic.c @@ -250,7 +250,40 @@ gst_gl_mosaic_callback (gpointer stuff) while (count < mosaic->input_frames->len && count < 6) { GstGLMixerFrameData *frame; - GLfloat *v_vertices; + /* *INDENT-OFF* */ + gfloat v_vertices = { + /* front face */ + 1.0f, 1.0f,-1.0f, 1.0f, 0.0f, + 1.0f,-1.0f,-1.0f, 1.0f, 1.0f, + -1.0f,-1.0f,-1.0f, 0.0f, 1.0f, + -1.0f, 1.0f,-1.0f, 0.0f, 0.0f, + /* right face */ + 1.0f, 1.0f, 1.0f, 1.0f, 0.0f, + 1.0f,-1.0f, 1.0f, 0.0f, 0.0f, + 1.0f,-1.0f,-1.0f, 0.0f, 1.0f, + 1.0f, 1.0f,-1.0f, 1.0f, 1.0f, + /* left face */ + -1.0f, 1.0f, 1.0f, 1.0f, 0.0f, + -1.0f, 1.0f,-1.0f, 1.0f, 1.0f, + -1.0f,-1.0f,-1.0f, 0.0f, 1.0f, + -1.0f,-1.0f, 1.0f, 0.0f, 0.0f, + /* top face */ + 1.0f,-1.0f, 1.0f, 1.0f, 0.0f, + -1.0f,-1.0f, 1.0f, 0.0f, 0.0f, + -1.0f,-1.0f,-1.0f, 0.0f, 1.0f, + 1.0f,-1.0f,-1.0f, 1.0f, 1.0f, + /* bottom face */ + 1.0f, 1.0f, 1.0f, 1.0f, 0.0f, + 1.0f, 1.0f,-1.0f, 1.0f, 1.0f, + -1.0f, 1.0f,-1.0f, 0.0f, 1.0f, + -1.0f, 1.0f, 1.0f, 0.0f, 0.0f, + /* back face */ + 1.0f, 1.0f, 1.0f, 1.0f, 0.0f, + -1.0f, 1.0f, 1.0f, 0.0f, 0.0f, + -1.0f,-1.0f, 1.0f, 0.0f, 1.0f, + 1.0f,-1.0f, 1.0f, 1.0f, 1.0f + }; + /* *INDENT-ON* */ guint in_tex; guint width, height; @@ -268,65 +301,6 @@ gst_gl_mosaic_callback (gpointer stuff) GST_TRACE ("processing texture:%u dimensions:%ux%u", in_tex, width, height); - /* *INDENT-OFF* */ - v_vertices = (GLfloat[]) { - /* front face */ - 1.0f, 1.0f, -1.0f, - 1.0f, 0.0f, - 1.0f, -1.0f, -1.0f, - 1.0f, 1.0f, - -1.0f, -1.0f, -1.0f, - 0.0f, 1.0f, - -1.0f, 1.0f, -1.0f, - 0.0f, 0.0f, - /* right face */ - 1.0f, 1.0f, 1.0f, - 1.0f, 0.0f, - 1.0f, -1.0f, 1.0f, - 0.0f, 0.0f, - 1.0f, -1.0f, -1.0f, - 0.0f, 1.0f, - 1.0f, 1.0f, -1.0f, - 1.0f, 1.0f, - /* left face */ - -1.0f, 1.0f, 1.0f, - 1.0f, 0.0f, - -1.0f, 1.0f, -1.0f, - 1.0f, 1.0f, - -1.0f, -1.0f, -1.0f, - 0.0f, 1.0f, - -1.0f, -1.0f, 1.0f, - 0.0f, 0.0f, - /* top face */ - 1.0f, -1.0f, 1.0f, - 1.0f, 0.0f, - -1.0f, -1.0f, 1.0f, - 0.0f, 0.0f, - -1.0f, -1.0f, -1.0f, - 0.0f, 1.0f, - 1.0f, -1.0f, -1.0f, - 1.0f, 1.0f, - /* bottom face */ - 1.0f, 1.0f, 1.0f, - 1.0f, 0.0f, - 1.0f, 1.0f, -1.0f, - 1.0f, 1.0f, - -1.0f, 1.0f, -1.0f, - 0.0f, 1.0f, - -1.0f, 1.0f, 1.0f, - 0.0f, 0.0f, - /* back face */ - 1.0f, 1.0f, 1.0f, - 1.0f, 0.0f, - -1.0f, 1.0f, 1.0f, - 0.0f, 0.0f, - -1.0f, -1.0f, 1.0f, - 0.0f, 1.0f, - 1.0f, -1.0f, 1.0f, - 1.0f, 1.0f - }; - /* *INDENT-ON* */ - gl->VertexAttribPointer (attr_position_loc, 3, GL_FLOAT, GL_FALSE, 5 * sizeof (GLfloat), &v_vertices[5 * 4 * count]); diff --git a/ext/gl/gstglvideomixer.c b/ext/gl/gstglvideomixer.c index d72f83ba7a..1e10bf3be2 100644 --- a/ext/gl/gstglvideomixer.c +++ b/ext/gl/gstglvideomixer.c @@ -221,22 +221,30 @@ gst_gl_video_mixer_callback (gpointer stuff) while (count < video_mixer->input_frames->len) { GstGLMixerFrameData *frame; - GLfloat *v_vertices; + /* *INDENT-OFF* */ + gfloat v_vertices = { + /* front face */ + -1.0,-1.0,-1.0f, 0.0f, 0.0f, + 1.0,-1.0,-1.0f, 1.0f, 0.0f, + 1.0, 1.0,-1.0f, 1.0f, 1.0f, + -1.0, 1.0,-1.0f, 0.0f, 1.0f, + }; + /* *INDENT-ON* */ guint in_tex; guint in_width, in_height; gfloat w, h; frame = g_ptr_array_index (video_mixer->input_frames, count); - in_tex = frame->texture; in_width = GST_VIDEO_INFO_WIDTH (&frame->pad->in_info); in_height = GST_VIDEO_INFO_HEIGHT (&frame->pad->in_info); - if (!frame || !in_tex || in_width <= 0 || in_height <= 0) { + if (!frame || !frame->texture || in_width <= 0 || in_height <= 0) { GST_DEBUG ("skipping texture:%u frame:%p width:%u height %u", in_tex, frame, in_width, in_height); count++; continue; } + in_tex = frame->texture; GST_TRACE ("processing texture:%u dimensions:%ux%u", in_tex, in_width, in_height); @@ -246,20 +254,6 @@ gst_gl_video_mixer_callback (gpointer stuff) GST_TRACE ("processing texture:%u dimensions:%ux%u, %fx%f", in_tex, in_width, in_height, w, h); - /* *INDENT-OFF* */ - v_vertices = (GLfloat[]) { - /* front face */ - -1.0, -1.0, -1.0f, - 0.0f, 0.0f, - 1.0, -1.0, -1.0f, - 1.0f, 0.0f, - 1.0, 1.0, -1.0f, - 1.0f, 1.0f, - -1.0, 1.0, -1.0f, - 0.0f, 1.0f, - }; - /* *INDENT-ON* */ - gl->VertexAttribPointer (attr_position_loc, 3, GL_FLOAT, GL_FALSE, 5 * sizeof (GLfloat), &v_vertices[0]); From 58206849b59cffa423699fe1c1c026acebd86713 Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Wed, 2 Apr 2014 23:05:11 +1100 Subject: [PATCH 003/381] gl: fix array initialization --- ext/gl/gstglmosaic.c | 2 +- ext/gl/gstglvideomixer.c | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ext/gl/gstglmosaic.c b/ext/gl/gstglmosaic.c index d5ff754199..8405c7ca49 100644 --- a/ext/gl/gstglmosaic.c +++ b/ext/gl/gstglmosaic.c @@ -251,7 +251,7 @@ gst_gl_mosaic_callback (gpointer stuff) while (count < mosaic->input_frames->len && count < 6) { GstGLMixerFrameData *frame; /* *INDENT-OFF* */ - gfloat v_vertices = { + gfloat v_vertices[] = { /* front face */ 1.0f, 1.0f,-1.0f, 1.0f, 0.0f, 1.0f,-1.0f,-1.0f, 1.0f, 1.0f, diff --git a/ext/gl/gstglvideomixer.c b/ext/gl/gstglvideomixer.c index 1e10bf3be2..118d767a4d 100644 --- a/ext/gl/gstglvideomixer.c +++ b/ext/gl/gstglvideomixer.c @@ -222,7 +222,7 @@ gst_gl_video_mixer_callback (gpointer stuff) while (count < video_mixer->input_frames->len) { GstGLMixerFrameData *frame; /* *INDENT-OFF* */ - gfloat v_vertices = { + gfloat v_vertices[] = { /* front face */ -1.0,-1.0,-1.0f, 0.0f, 0.0f, 1.0,-1.0,-1.0f, 1.0f, 0.0f, @@ -240,7 +240,7 @@ gst_gl_video_mixer_callback (gpointer stuff) if (!frame || !frame->texture || in_width <= 0 || in_height <= 0) { GST_DEBUG ("skipping texture:%u frame:%p width:%u height %u", - in_tex, frame, in_width, in_height); + frame->texture, frame, in_width, in_height); count++; continue; } From c2fbe354c1ef139418a2435e68e332434a4a1aa5 Mon Sep 17 00:00:00 2001 From: Vincent Penquerc'h Date: Tue, 8 Apr 2014 16:23:50 +0100 Subject: [PATCH 004/381] gl: test for frame NULLness before dereferencing it Coverity 1195172, 1195171 --- ext/gl/gstglmosaic.c | 7 ++++++- ext/gl/gstglvideomixer.c | 7 ++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/ext/gl/gstglmosaic.c b/ext/gl/gstglmosaic.c index 8405c7ca49..857c179931 100644 --- a/ext/gl/gstglmosaic.c +++ b/ext/gl/gstglmosaic.c @@ -288,11 +288,16 @@ gst_gl_mosaic_callback (gpointer stuff) guint width, height; frame = g_ptr_array_index (mosaic->input_frames, count); + if (!frame) { + GST_DEBUG ("skipping texture, null frame"); + count++; + continue; + } in_tex = frame->texture; width = GST_VIDEO_INFO_WIDTH (&frame->pad->in_info); height = GST_VIDEO_INFO_HEIGHT (&frame->pad->in_info); - if (!frame || !in_tex || width <= 0 || height <= 0) { + if (!in_tex || width <= 0 || height <= 0) { GST_DEBUG ("skipping texture:%u frame:%p width:%u height %u", in_tex, frame, width, height); count++; diff --git a/ext/gl/gstglvideomixer.c b/ext/gl/gstglvideomixer.c index 118d767a4d..b2aaeb9936 100644 --- a/ext/gl/gstglvideomixer.c +++ b/ext/gl/gstglvideomixer.c @@ -235,10 +235,15 @@ gst_gl_video_mixer_callback (gpointer stuff) gfloat w, h; frame = g_ptr_array_index (video_mixer->input_frames, count); + if (!frame) { + GST_DEBUG ("skipping texture, null frame"); + count++; + continue; + } in_width = GST_VIDEO_INFO_WIDTH (&frame->pad->in_info); in_height = GST_VIDEO_INFO_HEIGHT (&frame->pad->in_info); - if (!frame || !frame->texture || in_width <= 0 || in_height <= 0) { + if (!frame->texture || in_width <= 0 || in_height <= 0) { GST_DEBUG ("skipping texture:%u frame:%p width:%u height %u", frame->texture, frame, in_width, in_height); count++; From e3a48cb9d64e0545e42041ca1ab7f601d1fed40e Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Sun, 15 Jun 2014 12:26:21 +1000 Subject: [PATCH 005/381] glvideomixer: add positioning of input streams https://bugzilla.gnome.org/show_bug.cgi?id=729798 --- ext/gl/gstglvideomixer.c | 179 ++++++++++++++++++++++++++++++++++++--- ext/gl/gstglvideomixer.h | 1 + 2 files changed, 170 insertions(+), 10 deletions(-) diff --git a/ext/gl/gstglvideomixer.c b/ext/gl/gstglvideomixer.c index b2aaeb9936..b98107b9d3 100644 --- a/ext/gl/gstglvideomixer.c +++ b/ext/gl/gstglvideomixer.c @@ -72,12 +72,10 @@ static void gst_gl_video_mixer_callback (gpointer stuff); static const gchar *video_mixer_v_src = "attribute vec4 a_position; \n" "attribute vec2 a_texCoord; \n" - "uniform float x_scale; \n" - "uniform float y_scale; \n" "varying vec2 v_texCoord; \n" "void main() \n" "{ \n" - " gl_Position = a_position * vec4(x_scale, y_scale, 1.0, 1.0);\n" + " gl_Position = a_position; \n" " v_texCoord = a_texCoord; \n" "}"; /* fragment source */ @@ -90,6 +88,159 @@ static const gchar *video_mixer_f_src = " gl_FragColor = vec4(rgba.rgb, 1.0);\n" "} \n"; +#define GST_TYPE_GL_VIDEO_MIXER_PAD (gst_gl_video_mixer_pad_get_type()) +#define GST_GL_VIDEO_MIXER_PAD(obj) \ + (G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_GL_VIDEO_MIXER_PAD, GstGLVideoMixerPad)) +#define GST_GL_VIDEO_MIXER_PAD_CLASS(klass) \ + (G_TYPE_CHECK_CLASS_CAST((klass),GST_TYPE_GL_VIDEO_MIXER_PAD, GstGLVideoMixerPadClass)) +#define GST_IS_GL_VIDEO_MIXER_PAD(obj) \ + (G_TYPE_CHECK_INSTANCE_TYPE((obj),GST_TYPE_GL_VIDEO_MIXER_PAD)) +#define GST_IS_GL_VIDEO_MIXER_PAD_CLASS(klass) \ + (G_TYPE_CHECK_CLASS_TYPE((klass),GST_TYPE_GL_VIDEO_MIXER_PAD)) + +typedef struct _GstGLVideoMixerPad GstGLVideoMixerPad; +typedef struct _GstGLVideoMixerPadClass GstGLVideoMixerPadClass; +typedef struct _GstGLVideoMixerCollect GstGLVideoMixerCollect; + +/** + * GstGLVideoMixerPad: + * + * The opaque #GstGLVideoMixerPad structure. + */ +struct _GstGLVideoMixerPad +{ + GstGLMixerPad parent; + + /* < private > */ + /* properties */ + gint xpos, ypos; + guint zorder; + gdouble alpha; +}; + +struct _GstGLVideoMixerPadClass +{ + GstGLMixerPadClass parent_class; +}; + +GType gst_gl_video_mixer_pad_get_type (void); +G_DEFINE_TYPE (GstGLVideoMixerPad, gst_gl_video_mixer_pad, + GST_TYPE_GL_MIXER_PAD); + +static void gst_gl_video_mixer_pad_set_property (GObject * object, + guint prop_id, const GValue * value, GParamSpec * pspec); +static void gst_gl_video_mixer_pad_get_property (GObject * object, + guint prop_id, GValue * value, GParamSpec * pspec); + +#define DEFAULT_PAD_ZORDER 0 +#define DEFAULT_PAD_XPOS 0 +#define DEFAULT_PAD_YPOS 0 +#define DEFAULT_PAD_ALPHA 1.0 +enum +{ + PROP_PAD_0, + PROP_PAD_ZORDER, + PROP_PAD_XPOS, + PROP_PAD_YPOS, + PROP_PAD_ALPHA +}; + +static void +gst_gl_video_mixer_pad_init (GstGLVideoMixerPad * pad) +{ +} + +static void +gst_gl_video_mixer_pad_class_init (GstGLVideoMixerPadClass * klass) +{ + GObjectClass *gobject_class = (GObjectClass *) klass; + + gobject_class->set_property = gst_gl_video_mixer_pad_set_property; + gobject_class->get_property = gst_gl_video_mixer_pad_get_property; + + g_object_class_install_property (gobject_class, PROP_PAD_ZORDER, + g_param_spec_uint ("zorder", "Z-Order", "Z Order of the picture", + 0, 10000, DEFAULT_PAD_ZORDER, + G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); + g_object_class_install_property (gobject_class, PROP_PAD_XPOS, + g_param_spec_int ("xpos", "X Position", "X Position of the picture", + G_MININT, G_MAXINT, DEFAULT_PAD_XPOS, + G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); + g_object_class_install_property (gobject_class, PROP_PAD_YPOS, + g_param_spec_int ("ypos", "Y Position", "Y Position of the picture", + G_MININT, G_MAXINT, DEFAULT_PAD_YPOS, + G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); + g_object_class_install_property (gobject_class, PROP_PAD_ALPHA, + g_param_spec_double ("alpha", "Alpha", "Alpha of the picture", 0.0, 1.0, + DEFAULT_PAD_ALPHA, + G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); +} + +static void +gst_gl_video_mixer_pad_get_property (GObject * object, guint prop_id, + GValue * value, GParamSpec * pspec) +{ + GstGLVideoMixerPad *pad = GST_GL_VIDEO_MIXER_PAD (object); + + switch (prop_id) { + case PROP_PAD_ZORDER: + g_value_set_uint (value, pad->zorder); + break; + case PROP_PAD_XPOS: + g_value_set_int (value, pad->xpos); + break; + case PROP_PAD_YPOS: + g_value_set_int (value, pad->ypos); + break; + case PROP_PAD_ALPHA: + g_value_set_double (value, pad->alpha); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static int +pad_zorder_compare (const GstGLVideoMixerPad * pad1, + const GstGLVideoMixerPad * pad2) +{ + return pad1->zorder - pad2->zorder; +} + +static void +gst_gl_video_mixer_pad_set_property (GObject * object, guint prop_id, + const GValue * value, GParamSpec * pspec) +{ + GstGLVideoMixerPad *pad = GST_GL_VIDEO_MIXER_PAD (object); + GstGLMixer *mix = GST_GL_MIXER (gst_pad_get_parent (GST_PAD (pad))); + + switch (prop_id) { + case PROP_PAD_ZORDER: + GST_GL_MIXER_LOCK (mix); + pad->zorder = g_value_get_uint (value); + + mix->sinkpads = g_slist_sort (mix->sinkpads, + (GCompareFunc) pad_zorder_compare); + GST_GL_MIXER_UNLOCK (mix); + break; + case PROP_PAD_XPOS: + pad->xpos = g_value_get_int (value); + break; + case PROP_PAD_YPOS: + pad->ypos = g_value_get_int (value); + break; + case PROP_PAD_ALPHA: + pad->alpha = g_value_get_double (value); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } + + gst_object_unref (mix); +} + static void gst_gl_video_mixer_class_init (GstGLVideoMixerClass * klass) { @@ -117,6 +268,9 @@ gst_gl_video_mixer_init (GstGLVideoMixer * video_mixer) { video_mixer->shader = NULL; video_mixer->input_frames = NULL; + + gst_gl_mixer_set_pad_type (GST_GL_MIXER (video_mixer), + GST_TYPE_GL_VIDEO_MIXER_PAD); } static void @@ -221,6 +375,7 @@ gst_gl_video_mixer_callback (gpointer stuff) while (count < video_mixer->input_frames->len) { GstGLMixerFrameData *frame; + GstGLVideoMixerPad *pad; /* *INDENT-OFF* */ gfloat v_vertices[] = { /* front face */ @@ -240,6 +395,7 @@ gst_gl_video_mixer_callback (gpointer stuff) count++; continue; } + pad = (GstGLVideoMixerPad *) frame->pad; in_width = GST_VIDEO_INFO_WIDTH (&frame->pad->in_info); in_height = GST_VIDEO_INFO_HEIGHT (&frame->pad->in_info); @@ -251,13 +407,18 @@ gst_gl_video_mixer_callback (gpointer stuff) } in_tex = frame->texture; - GST_TRACE ("processing texture:%u dimensions:%ux%u", in_tex, in_width, - in_height); - w = ((gfloat) in_width / (gfloat) out_width); h = ((gfloat) in_height / (gfloat) out_height); - GST_TRACE ("processing texture:%u dimensions:%ux%u, %fx%f", in_tex, - in_width, in_height, w, h); + + v_vertices[0] = v_vertices[15] = + 2.0f * (gfloat) pad->xpos / (gfloat) out_width - 1.0f; + v_vertices[1] = v_vertices[6] = + 2.0f * (gfloat) pad->ypos / (gfloat) out_height - 1.0f; + v_vertices[5] = v_vertices[10] = v_vertices[0] + 2.0f * w; + v_vertices[11] = v_vertices[16] = v_vertices[1] + 2.0f * h; + GST_TRACE ("processing texture:%u dimensions:%ux%u, at %f,%f %fx%f", in_tex, + in_width, in_height, v_vertices[0], v_vertices[1], v_vertices[5], + v_vertices[11]); gl->VertexAttribPointer (attr_position_loc, 3, GL_FLOAT, GL_FALSE, 5 * sizeof (GLfloat), &v_vertices[0]); @@ -274,8 +435,6 @@ gst_gl_video_mixer_callback (gpointer stuff) gl->ActiveTexture (GL_TEXTURE0); gl->BindTexture (GL_TEXTURE_2D, in_tex); gst_gl_shader_set_uniform_1i (video_mixer->shader, "texture", 0); - gst_gl_shader_set_uniform_1f (video_mixer->shader, "x_scale", w); - gst_gl_shader_set_uniform_1f (video_mixer->shader, "y_scale", h); gl->DrawElements (GL_TRIANGLES, 6, GL_UNSIGNED_SHORT, indices); diff --git a/ext/gl/gstglvideomixer.h b/ext/gl/gstglvideomixer.h index 716c60a7e0..69a077100c 100644 --- a/ext/gl/gstglvideomixer.h +++ b/ext/gl/gstglvideomixer.h @@ -22,6 +22,7 @@ #define _GST_GL_VIDEO_MIXER_H_ #include +#include G_BEGIN_DECLS From ec7df9773fcb9b5b98841d6039e17a2aff78c6c3 Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Sun, 15 Jun 2014 13:44:04 +1000 Subject: [PATCH 006/381] glvideomixer: support input frame scaling --- ext/gl/gstglvideomixer.c | 36 ++++++++++++++++++++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/ext/gl/gstglvideomixer.c b/ext/gl/gstglvideomixer.c index b98107b9d3..d27d119a61 100644 --- a/ext/gl/gstglvideomixer.c +++ b/ext/gl/gstglvideomixer.c @@ -114,6 +114,7 @@ struct _GstGLVideoMixerPad /* < private > */ /* properties */ gint xpos, ypos; + gint width, height; guint zorder; gdouble alpha; }; @@ -135,6 +136,8 @@ static void gst_gl_video_mixer_pad_get_property (GObject * object, #define DEFAULT_PAD_ZORDER 0 #define DEFAULT_PAD_XPOS 0 #define DEFAULT_PAD_YPOS 0 +#define DEFAULT_PAD_WIDTH 0 +#define DEFAULT_PAD_HEIGHT 0 #define DEFAULT_PAD_ALPHA 1.0 enum { @@ -142,6 +145,8 @@ enum PROP_PAD_ZORDER, PROP_PAD_XPOS, PROP_PAD_YPOS, + PROP_PAD_WIDTH, + PROP_PAD_HEIGHT, PROP_PAD_ALPHA }; @@ -170,6 +175,14 @@ gst_gl_video_mixer_pad_class_init (GstGLVideoMixerPadClass * klass) g_param_spec_int ("ypos", "Y Position", "Y Position of the picture", G_MININT, G_MAXINT, DEFAULT_PAD_YPOS, G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); + g_object_class_install_property (gobject_class, PROP_PAD_WIDTH, + g_param_spec_int ("width", "Width", "Width of the picture", + G_MININT, G_MAXINT, DEFAULT_PAD_WIDTH, + G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); + g_object_class_install_property (gobject_class, PROP_PAD_HEIGHT, + g_param_spec_int ("height", "Height", "Height of the picture", + G_MININT, G_MAXINT, DEFAULT_PAD_HEIGHT, + G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); g_object_class_install_property (gobject_class, PROP_PAD_ALPHA, g_param_spec_double ("alpha", "Alpha", "Alpha of the picture", 0.0, 1.0, DEFAULT_PAD_ALPHA, @@ -192,6 +205,12 @@ gst_gl_video_mixer_pad_get_property (GObject * object, guint prop_id, case PROP_PAD_YPOS: g_value_set_int (value, pad->ypos); break; + case PROP_PAD_WIDTH: + g_value_set_int (value, pad->width); + break; + case PROP_PAD_HEIGHT: + g_value_set_int (value, pad->height); + break; case PROP_PAD_ALPHA: g_value_set_double (value, pad->alpha); break; @@ -230,6 +249,12 @@ gst_gl_video_mixer_pad_set_property (GObject * object, guint prop_id, case PROP_PAD_YPOS: pad->ypos = g_value_get_int (value); break; + case PROP_PAD_WIDTH: + pad->width = g_value_get_int (value); + break; + case PROP_PAD_HEIGHT: + pad->height = g_value_get_int (value); + break; case PROP_PAD_ALPHA: pad->alpha = g_value_get_double (value); break; @@ -387,6 +412,7 @@ gst_gl_video_mixer_callback (gpointer stuff) /* *INDENT-ON* */ guint in_tex; guint in_width, in_height; + guint pad_width, pad_height; gfloat w, h; frame = g_ptr_array_index (video_mixer->input_frames, count); @@ -406,15 +432,21 @@ gst_gl_video_mixer_callback (gpointer stuff) continue; } in_tex = frame->texture; + pad_width = pad->width <= 0 ? in_width : pad->width; + pad_height = pad->height <= 0 ? in_height : pad->height; - w = ((gfloat) in_width / (gfloat) out_width); - h = ((gfloat) in_height / (gfloat) out_height); + w = ((gfloat) pad_width / (gfloat) out_width); + h = ((gfloat) pad_height / (gfloat) out_height); + /* top-left */ v_vertices[0] = v_vertices[15] = 2.0f * (gfloat) pad->xpos / (gfloat) out_width - 1.0f; + /* bottom-left */ v_vertices[1] = v_vertices[6] = 2.0f * (gfloat) pad->ypos / (gfloat) out_height - 1.0f; + /* top-right */ v_vertices[5] = v_vertices[10] = v_vertices[0] + 2.0f * w; + /* bottom-right */ v_vertices[11] = v_vertices[16] = v_vertices[1] + 2.0f * h; GST_TRACE ("processing texture:%u dimensions:%ux%u, at %f,%f %fx%f", in_tex, in_width, in_height, v_vertices[0], v_vertices[1], v_vertices[5], From fabdcd1dfc8c13a117be07f38ce2ea0b4785d866 Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Sun, 15 Jun 2014 13:59:07 +1000 Subject: [PATCH 007/381] glvideomixer: wire up the alpha pad property --- ext/gl/gstglvideomixer.c | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/ext/gl/gstglvideomixer.c b/ext/gl/gstglvideomixer.c index d27d119a61..c1c76c0b1a 100644 --- a/ext/gl/gstglvideomixer.c +++ b/ext/gl/gstglvideomixer.c @@ -81,11 +81,12 @@ static const gchar *video_mixer_v_src = /* fragment source */ static const gchar *video_mixer_f_src = "uniform sampler2D texture; \n" + "uniform float alpha;\n" "varying vec2 v_texCoord; \n" "void main() \n" "{ \n" " vec4 rgba = texture2D( texture, v_texCoord );\n" - " gl_FragColor = vec4(rgba.rgb, 1.0);\n" + " gl_FragColor = vec4(rgba.rgb, rgba.a * alpha);\n" "} \n"; #define GST_TYPE_GL_VIDEO_MIXER_PAD (gst_gl_video_mixer_pad_get_type()) @@ -153,6 +154,7 @@ enum static void gst_gl_video_mixer_pad_init (GstGLVideoMixerPad * pad) { + pad->alpha = 1.0; } static void @@ -448,9 +450,9 @@ gst_gl_video_mixer_callback (gpointer stuff) v_vertices[5] = v_vertices[10] = v_vertices[0] + 2.0f * w; /* bottom-right */ v_vertices[11] = v_vertices[16] = v_vertices[1] + 2.0f * h; - GST_TRACE ("processing texture:%u dimensions:%ux%u, at %f,%f %fx%f", in_tex, - in_width, in_height, v_vertices[0], v_vertices[1], v_vertices[5], - v_vertices[11]); + GST_TRACE ("processing texture:%u dimensions:%ux%u, at %f,%f %fx%f with " + "alpha", in_tex, in_width, in_height, v_vertices[0], v_vertices[1], + v_vertices[5], v_vertices[11], pad->alpha); gl->VertexAttribPointer (attr_position_loc, 3, GL_FLOAT, GL_FALSE, 5 * sizeof (GLfloat), &v_vertices[0]); @@ -461,12 +463,14 @@ gst_gl_video_mixer_callback (gpointer stuff) gl->EnableVertexAttribArray (attr_position_loc); gl->EnableVertexAttribArray (attr_texture_loc); - gl->BlendFunc (GL_SRC_COLOR, GL_ONE_MINUS_SRC_COLOR); - gl->BlendEquation (GL_FUNC_ADD); + gl->BlendFuncSeparate (GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA, GL_ONE, + GL_ZERO); + gl->BlendEquationSeparate (GL_FUNC_ADD, GL_FUNC_ADD); gl->ActiveTexture (GL_TEXTURE0); gl->BindTexture (GL_TEXTURE_2D, in_tex); gst_gl_shader_set_uniform_1i (video_mixer->shader, "texture", 0); + gst_gl_shader_set_uniform_1f (video_mixer->shader, "alpha", pad->alpha); gl->DrawElements (GL_TRIANGLES, 6, GL_UNSIGNED_SHORT, indices); From b1a11258b15bffe09fc9cbec796e84db51e3319f Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Sun, 15 Jun 2014 15:18:46 +1000 Subject: [PATCH 008/381] glvideomixer: silence incorrect number of arguments in format warning --- ext/gl/gstglvideomixer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/gl/gstglvideomixer.c b/ext/gl/gstglvideomixer.c index c1c76c0b1a..e3d5807410 100644 --- a/ext/gl/gstglvideomixer.c +++ b/ext/gl/gstglvideomixer.c @@ -451,7 +451,7 @@ gst_gl_video_mixer_callback (gpointer stuff) /* bottom-right */ v_vertices[11] = v_vertices[16] = v_vertices[1] + 2.0f * h; GST_TRACE ("processing texture:%u dimensions:%ux%u, at %f,%f %fx%f with " - "alpha", in_tex, in_width, in_height, v_vertices[0], v_vertices[1], + "alpha:%f", in_tex, in_width, in_height, v_vertices[0], v_vertices[1], v_vertices[5], v_vertices[11], pad->alpha); gl->VertexAttribPointer (attr_position_loc, 3, GL_FLOAT, From a7289ae6062044053758902b43c6919c6cca5119 Mon Sep 17 00:00:00 2001 From: Mathieu Duponchelle Date: Tue, 3 Jun 2014 19:00:34 +0200 Subject: [PATCH 009/381] videoaggregator: Create a new GstVideoAggregator baseclass This base class has been added to a newly created libgstbadvideo library Co-Authored by: Thibault Saunier https://bugzilla.gnome.org/show_bug.cgi?id=731918 --- gst-libs/gst/video/gstvideoaggregator.c | 1928 +++++++++++++++++++++++ gst-libs/gst/video/gstvideoaggregator.h | 109 ++ 2 files changed, 2037 insertions(+) create mode 100644 gst-libs/gst/video/gstvideoaggregator.c create mode 100644 gst-libs/gst/video/gstvideoaggregator.h diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c new file mode 100644 index 0000000000..06d80e19d6 --- /dev/null +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -0,0 +1,1928 @@ +/* Generic video aggregator plugin + * Copyright (C) 2004, 2008 Wim Taymans + * Copyright (C) 2010 Sebastian Dröge + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +/** + * SECTION:gstvideoaggregator + * @short_description: Base class for video aggregators + * + * VideoAggregator can accept AYUV, ARGB and BGRA video streams. For each of the requested + * sink pads it will compare the incoming geometry and framerate to define the + * output parameters. Indeed output video frames will have the geometry of the + * biggest incoming video stream and the framerate of the fastest incoming one. + * + * VideoAggregator will do colorspace conversion. + * + * Zorder for each input stream can be configured on the + * #GstVideoAggregatorPad. + * + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include + +#include "gstvideoaggregator.h" +#include "gstvideoaggregatorpad.h" +#include "videoconvert.h" + +GST_DEBUG_CATEGORY_STATIC (gst_videoaggregator_debug); +#define GST_CAT_DEFAULT gst_videoaggregator_debug + +/* Needed prototypes */ +static void gst_videoaggregator_reset_qos (GstVideoAggregator * vagg); + +/**************************************** + * GstVideoAggregatorPad implementation * + ****************************************/ + +#define DEFAULT_PAD_ZORDER 0 +enum +{ + PROP_PAD_0, + PROP_PAD_ZORDER, +}; + +G_DEFINE_TYPE (GstVideoAggregatorPad, gst_videoaggregator_pad, + GST_TYPE_AGGREGATOR_PAD); + +static void +gst_videoaggregator_pad_get_property (GObject * object, guint prop_id, + GValue * value, GParamSpec * pspec) +{ + GstVideoAggregatorPad *pad = GST_VIDEO_AGGREGATOR_PAD (object); + + switch (prop_id) { + case PROP_PAD_ZORDER: + g_value_set_uint (value, pad->zorder); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static int +pad_zorder_compare (const GstVideoAggregatorPad * pad1, + const GstVideoAggregatorPad * pad2) +{ + return pad1->zorder - pad2->zorder; +} + +static void +gst_videoaggregator_pad_set_property (GObject * object, guint prop_id, + const GValue * value, GParamSpec * pspec) +{ + GstVideoAggregatorPad *pad = GST_VIDEO_AGGREGATOR_PAD (object); + GstVideoAggregator *vagg = + GST_VIDEO_AGGREGATOR (gst_pad_get_parent (GST_PAD (pad))); + + switch (prop_id) { + case PROP_PAD_ZORDER: + GST_OBJECT_LOCK (vagg); + pad->zorder = g_value_get_uint (value); + GST_ELEMENT (vagg)->sinkpads = g_list_sort (GST_ELEMENT (vagg)->sinkpads, + (GCompareFunc) pad_zorder_compare); + GST_OBJECT_UNLOCK (vagg); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } + + gst_object_unref (vagg); +} + +static gboolean +_flush_pad (GstAggregatorPad * aggpad, GstAggregator * aggregator) +{ + GstVideoAggregator *vagg = GST_VIDEO_AGGREGATOR (aggregator); + GstVideoAggregatorPad *pad = GST_VIDEO_AGGREGATOR_PAD (aggpad); + + gst_videoaggregator_reset_qos (vagg); + gst_buffer_replace (&pad->buffer, NULL); + pad->start_time = -1; + pad->end_time = -1; + + return TRUE; +} + +static void +gst_videoaggregator_pad_finalize (GObject * o) +{ + GstVideoAggregatorPad *vaggpad = GST_VIDEO_AGGREGATOR_PAD (o); + + if (vaggpad->convert) + videoconvert_convert_free (vaggpad->convert); + vaggpad->convert = NULL; + + G_OBJECT_CLASS (gst_videoaggregator_pad_parent_class)->dispose (o); +} + +static void +gst_videoaggregator_pad_class_init (GstVideoAggregatorPadClass * klass) +{ + GObjectClass *gobject_class = (GObjectClass *) klass; + GstAggregatorPadClass *aggpadclass = (GstAggregatorPadClass *) klass; + + gobject_class->set_property = gst_videoaggregator_pad_set_property; + gobject_class->get_property = gst_videoaggregator_pad_get_property; + gobject_class->finalize = gst_videoaggregator_pad_finalize; + + g_object_class_install_property (gobject_class, PROP_PAD_ZORDER, + g_param_spec_uint ("zorder", "Z-Order", "Z Order of the picture", + 0, 10000, DEFAULT_PAD_ZORDER, + G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); + + aggpadclass->flush = GST_DEBUG_FUNCPTR (_flush_pad); +} + +static void +gst_videoaggregator_pad_init (GstVideoAggregatorPad * vaggpad) +{ + vaggpad->zorder = DEFAULT_PAD_ZORDER; + vaggpad->convert = NULL; + vaggpad->need_conversion_update = FALSE; + vaggpad->aggregated_frame = NULL; + vaggpad->converted_buffer = NULL; +} + +/********************************* + * GstChildProxy implementation * + *********************************/ +static GObject * +gst_videoaggregator_child_proxy_get_child_by_index (GstChildProxy * child_proxy, + guint index) +{ + GObject *obj; + GstVideoAggregator *vagg = GST_VIDEO_AGGREGATOR (child_proxy); + + GST_OBJECT_LOCK (vagg); + if ((obj = g_list_nth_data (GST_ELEMENT (vagg)->sinkpads, index))) + g_object_ref (obj); + GST_OBJECT_UNLOCK (vagg); + + return obj; +} + +static guint +gst_videoaggregator_child_proxy_get_children_count (GstChildProxy * child_proxy) +{ + guint count = 0; + + GST_OBJECT_LOCK (child_proxy); + count = GST_ELEMENT (child_proxy)->numsinkpads; + GST_OBJECT_UNLOCK (child_proxy); + + GST_INFO_OBJECT (child_proxy, "Children Count: %d", count); + + return count; +} + +static void +gst_videoaggregator_child_proxy_init (gpointer g_iface, gpointer iface_data) +{ + GstChildProxyInterface *iface = g_iface; + + GST_INFO ("intializing child proxy interface"); + iface->get_child_by_index = + gst_videoaggregator_child_proxy_get_child_by_index; + iface->get_children_count = + gst_videoaggregator_child_proxy_get_children_count; +} + +/************************************** + * GstVideoAggregator implementation * + **************************************/ + +#define GST_VIDEO_AGGREGATOR_GET_LOCK(vagg) (&GST_VIDEO_AGGREGATOR(vagg)->priv->lock) + +#define GST_VIDEO_AGGREGATOR_LOCK(vagg) G_STMT_START { \ + GST_LOG_OBJECT (vagg, "Taking EVENT lock from thread %p", \ + g_thread_self()); \ + g_mutex_lock(GST_VIDEO_AGGREGATOR_GET_LOCK(vagg)); \ + GST_LOG_OBJECT (vagg, "Took EVENT lock from thread %p", \ + g_thread_self()); \ + } G_STMT_END + +#define GST_VIDEO_AGGREGATOR_UNLOCK(vagg) G_STMT_START { \ + GST_LOG_OBJECT (vagg, "Releasing EVENT lock from thread %p", \ + g_thread_self()); \ + g_mutex_unlock(GST_VIDEO_AGGREGATOR_GET_LOCK(vagg)); \ + GST_LOG_OBJECT (vagg, "Took EVENT lock from thread %p", \ + g_thread_self()); \ + } G_STMT_END + + +#define GST_VIDEO_AGGREGATOR_GET_SETCAPS_LOCK(vagg) (&GST_VIDEO_AGGREGATOR(vagg)->priv->setcaps_lock) +#define GST_VIDEO_AGGREGATOR_SETCAPS_LOCK(vagg) G_STMT_START { \ + GST_LOG_OBJECT (vagg, "Taking SETCAPS lock from thread %p", \ + g_thread_self()); \ + g_mutex_lock(GST_VIDEO_AGGREGATOR_GET_SETCAPS_LOCK(vagg)); \ + GST_LOG_OBJECT (vagg, "Took SETCAPS lock from thread %p", \ + g_thread_self()); \ + } G_STMT_END + +#define GST_VIDEO_AGGREGATOR_SETCAPS_UNLOCK(vagg) G_STMT_START { \ + GST_LOG_OBJECT (vagg, "Releasing SETCAPS lock from thread %p", \ + g_thread_self()); \ + g_mutex_unlock(GST_VIDEO_AGGREGATOR_GET_SETCAPS_LOCK(vagg)); \ + GST_LOG_OBJECT (vagg, "Took SETCAPS lock from thread %p", \ + g_thread_self()); \ + } G_STMT_END + +struct _GstVideoAggregatorPrivate +{ + /* Lock to prevent the state to change while aggregating */ + GMutex lock; + + /* Lock to prevent two src setcaps from happening at the same time */ + GMutex setcaps_lock; + + /* Current downstream segment */ + GstClockTime ts_offset; + guint64 nframes; + + /* QoS stuff */ + gdouble proportion; + GstClockTime earliest_time; + guint64 qos_processed, qos_dropped; + + /* current caps */ + GstCaps *current_caps; + gboolean send_caps; +}; + +G_DEFINE_ABSTRACT_TYPE_WITH_CODE (GstVideoAggregator, gst_videoaggregator, + GST_TYPE_AGGREGATOR, G_IMPLEMENT_INTERFACE (GST_TYPE_CHILD_PROXY, + gst_videoaggregator_child_proxy_init)); + +static void +_find_best_video_format (GstVideoAggregator * vagg, GstCaps * downstream_caps, + GstVideoInfo * best_info, GstVideoFormat * best_format, + gboolean * at_least_one_alpha) +{ + GList *tmp; + GstCaps *possible_caps; + GstVideoAggregatorPad *pad; + gboolean need_alpha = FALSE; + gint best_format_number = 0; + GHashTable *formats_table = g_hash_table_new (g_direct_hash, g_direct_equal); + + GST_OBJECT_LOCK (vagg); + for (tmp = GST_ELEMENT (vagg)->sinkpads; tmp; tmp = tmp->next) { + GstStructure *s; + gint format_number; + + pad = tmp->data; + + if (!pad->info.finfo) + continue; + + if (pad->info.finfo->flags & GST_VIDEO_FORMAT_FLAG_ALPHA) + *at_least_one_alpha = TRUE; + + /* If we want alpha, disregard all the other formats */ + if (need_alpha && !(pad->info.finfo->flags & GST_VIDEO_FORMAT_FLAG_ALPHA)) + continue; + + /* This can happen if we release a pad and another pad hasn't been negotiated_caps yet */ + if (GST_VIDEO_INFO_FORMAT (&pad->info) == GST_VIDEO_FORMAT_UNKNOWN) + continue; + + possible_caps = gst_video_info_to_caps (&pad->info); + + s = gst_caps_get_structure (possible_caps, 0); + gst_structure_remove_fields (s, "width", "height", "framerate", + "pixel-aspect-ratio", "interlace-mode", NULL); + + /* Can downstream accept this format ? */ + if (!gst_caps_can_intersect (downstream_caps, possible_caps)) { + gst_caps_unref (possible_caps); + continue; + } + + gst_caps_unref (possible_caps); + + format_number = + GPOINTER_TO_INT (g_hash_table_lookup (formats_table, + GINT_TO_POINTER (GST_VIDEO_INFO_FORMAT (&pad->info)))); + format_number += 1; + + g_hash_table_replace (formats_table, + GINT_TO_POINTER (GST_VIDEO_INFO_FORMAT (&pad->info)), + GINT_TO_POINTER (format_number)); + + /* If that pad is the first with alpha, set it as the new best format */ + if (!need_alpha && (pad->info.finfo->flags & GST_VIDEO_FORMAT_FLAG_ALPHA)) { + need_alpha = TRUE; + *best_format = GST_VIDEO_INFO_FORMAT (&pad->info); + *best_info = pad->info; + best_format_number = format_number; + } else if (format_number > best_format_number) { + *best_format = GST_VIDEO_INFO_FORMAT (&pad->info); + *best_info = pad->info; + best_format_number = format_number; + } + } + GST_OBJECT_UNLOCK (vagg); + + g_hash_table_unref (formats_table); +} + +static gboolean +gst_videoaggregator_update_converters (GstVideoAggregator * vagg) +{ + GList *tmp; + GstVideoAggregatorPad *pad; + GstVideoFormat best_format; + GstVideoInfo best_info; + gboolean at_least_one_alpha = FALSE; + GstCaps *downstream_caps; + gchar *best_colorimetry; + const gchar *best_chroma; + GstElementClass *klass = GST_ELEMENT_GET_CLASS (vagg); + GstVideoAggregatorClass *vagg_klass = (GstVideoAggregatorClass *) klass; + GstAggregator *agg = GST_AGGREGATOR (vagg); + + best_format = GST_VIDEO_FORMAT_UNKNOWN; + gst_video_info_init (&best_info); + + downstream_caps = gst_pad_get_allowed_caps (agg->srcpad); + + if (!downstream_caps || gst_caps_is_empty (downstream_caps)) { + GST_INFO_OBJECT (vagg, "No downstream caps found %" + GST_PTR_FORMAT, downstream_caps); + return FALSE; + } + + + if (vagg_klass->disable_frame_conversion == FALSE) + _find_best_video_format (vagg, downstream_caps, &best_info, &best_format, + &at_least_one_alpha); + + if (best_format == GST_VIDEO_FORMAT_UNKNOWN) { + downstream_caps = gst_caps_fixate (downstream_caps); + gst_video_info_from_caps (&best_info, downstream_caps); + best_format = GST_VIDEO_INFO_FORMAT (&best_info); + } + + gst_caps_unref (downstream_caps); + + if (at_least_one_alpha + && !(best_info.finfo->flags & GST_VIDEO_FORMAT_FLAG_ALPHA)) { + GST_ELEMENT_ERROR (vagg, CORE, NEGOTIATION, + ("At least one of the input pads contains alpha, but downstream can't support alpha."), + ("Either convert your inputs to not contain alpha or add a videoconvert after the aggregator")); + return FALSE; + } + + best_colorimetry = gst_video_colorimetry_to_string (&(best_info.colorimetry)); + best_chroma = gst_video_chroma_to_string (best_info.chroma_site); + vagg->info = best_info; + + GST_DEBUG_OBJECT (vagg, + "The output format will now be : %d with colorimetry : %s and chroma : %s", + best_format, best_colorimetry, best_chroma); + + /* Then browse the sinks once more, setting or unsetting conversion if needed */ + GST_OBJECT_LOCK (vagg); + for (tmp = GST_ELEMENT (vagg)->sinkpads; tmp; tmp = tmp->next) { + gchar *colorimetry; + const gchar *chroma; + + pad = tmp->data; + + if (!pad->info.finfo) + continue; + + if (GST_VIDEO_INFO_FORMAT (&pad->info) == GST_VIDEO_FORMAT_UNKNOWN) + continue; + + if (pad->convert) + videoconvert_convert_free (pad->convert); + + pad->convert = NULL; + + colorimetry = gst_video_colorimetry_to_string (&(pad->info.colorimetry)); + chroma = gst_video_chroma_to_string (pad->info.chroma_site); + + if (best_format != GST_VIDEO_INFO_FORMAT (&pad->info) || + g_strcmp0 (colorimetry, best_colorimetry) || + g_strcmp0 (chroma, best_chroma)) { + GST_DEBUG_OBJECT (pad, "This pad will be converted from %d to %d", + GST_VIDEO_INFO_FORMAT (&pad->info), + GST_VIDEO_INFO_FORMAT (&best_info)); + pad->convert = videoconvert_convert_new (&pad->info, &best_info); + pad->need_conversion_update = TRUE; + if (!pad->convert) { + g_free (colorimetry); + g_free (best_colorimetry); + GST_WARNING ("No path found for conversion"); + GST_OBJECT_UNLOCK (vagg); + return FALSE; + } + } else { + GST_DEBUG_OBJECT (pad, "This pad will not need conversion"); + } + g_free (colorimetry); + } + GST_OBJECT_UNLOCK (vagg); + + g_free (best_colorimetry); + return TRUE; +} + +static gboolean +gst_videoaggregator_src_setcaps (GstVideoAggregator * vagg, GstCaps * caps) +{ + GstAggregator *agg = GST_AGGREGATOR (vagg); + gboolean ret = FALSE; + GstVideoInfo info; + + GstPad *pad = GST_AGGREGATOR (vagg)->srcpad; + + GST_INFO_OBJECT (pad, "set src caps: %" GST_PTR_FORMAT, caps); + + if (!gst_video_info_from_caps (&info, caps)) + goto done; + + ret = TRUE; + + GST_VIDEO_AGGREGATOR_LOCK (vagg); + + if (GST_VIDEO_INFO_FPS_N (&vagg->info) != GST_VIDEO_INFO_FPS_N (&info) || + GST_VIDEO_INFO_FPS_D (&vagg->info) != GST_VIDEO_INFO_FPS_D (&info)) { + if (agg->segment.position != -1) { + vagg->priv->ts_offset = agg->segment.position - agg->segment.start; + vagg->priv->nframes = 0; + } + gst_videoaggregator_reset_qos (vagg); + } + + vagg->info = info; + + GST_VIDEO_AGGREGATOR_UNLOCK (vagg); + + if (vagg->priv->current_caps == NULL || + gst_caps_is_equal (caps, vagg->priv->current_caps) == FALSE) { + gst_caps_replace (&vagg->priv->current_caps, caps); + vagg->priv->send_caps = TRUE; + } + +done: + return ret; +} + +static gboolean +gst_videoaggregator_update_src_caps (GstVideoAggregator * vagg) +{ + GList *l; + gint best_width = -1, best_height = -1; + gdouble best_fps = -1, cur_fps; + gint best_fps_n = -1, best_fps_d = -1; + gboolean ret = TRUE; + GstElementClass *klass = GST_ELEMENT_GET_CLASS (vagg); + GstVideoAggregatorClass *vagg_klass = (GstVideoAggregatorClass *) klass; + GstAggregator *agg = GST_AGGREGATOR (vagg); + + GST_VIDEO_AGGREGATOR_SETCAPS_LOCK (vagg); + GST_VIDEO_AGGREGATOR_LOCK (vagg); + GST_OBJECT_LOCK (vagg); + for (l = GST_ELEMENT (vagg)->sinkpads; l; l = l->next) { + GstVideoAggregatorPad *mpad = l->data; + gint this_width, this_height; + gint fps_n, fps_d; + gint width, height; + + fps_n = GST_VIDEO_INFO_FPS_N (&mpad->info); + fps_d = GST_VIDEO_INFO_FPS_D (&mpad->info); + width = GST_VIDEO_INFO_WIDTH (&mpad->info); + height = GST_VIDEO_INFO_HEIGHT (&mpad->info); + + if (width == 0 || height == 0) + continue; + + this_width = width; + this_height = height; + + if (best_width < this_width) + best_width = this_width; + if (best_height < this_height) + best_height = this_height; + + if (fps_d == 0) + cur_fps = 0.0; + else + gst_util_fraction_to_double (fps_n, fps_d, &cur_fps); + + if (best_fps < cur_fps) { + best_fps = cur_fps; + best_fps_n = fps_n; + best_fps_d = fps_d; + } + } + GST_OBJECT_UNLOCK (vagg); + + if (best_fps_n <= 0 || best_fps_d <= 0 || best_fps == 0.0) { + best_fps_n = 25; + best_fps_d = 1; + best_fps = 25.0; + } + + if (best_width > 0 && best_height > 0 && best_fps > 0) { + GstCaps *caps, *peercaps; + GstStructure *s; + GstVideoInfo info; + + if (GST_VIDEO_INFO_FPS_N (&vagg->info) != best_fps_n || + GST_VIDEO_INFO_FPS_D (&vagg->info) != best_fps_d) { + if (agg->segment.position != -1) { + vagg->priv->ts_offset = agg->segment.position - agg->segment.start; + vagg->priv->nframes = 0; + } + } + gst_video_info_init (&info); + gst_video_info_set_format (&info, GST_VIDEO_INFO_FORMAT (&vagg->info), + best_width, best_height); + info.fps_n = best_fps_n; + info.fps_d = best_fps_d; + info.par_n = GST_VIDEO_INFO_PAR_N (&vagg->info); + info.par_d = GST_VIDEO_INFO_PAR_D (&vagg->info); + + if (vagg_klass->update_info) { + if (!vagg_klass->update_info (vagg, &info)) { + ret = FALSE; + GST_VIDEO_AGGREGATOR_UNLOCK (vagg); + goto done; + } + } + + caps = gst_video_info_to_caps (&info); + + peercaps = gst_pad_peer_query_caps (agg->srcpad, NULL); + if (peercaps) { + GstCaps *tmp; + + s = gst_caps_get_structure (caps, 0); + gst_structure_set (s, "width", GST_TYPE_INT_RANGE, 1, G_MAXINT, "height", + GST_TYPE_INT_RANGE, 1, G_MAXINT, "framerate", GST_TYPE_FRACTION_RANGE, + 0, 1, G_MAXINT, 1, NULL); + + tmp = gst_caps_intersect (caps, peercaps); + gst_caps_unref (caps); + gst_caps_unref (peercaps); + caps = tmp; + if (gst_caps_is_empty (caps)) { + GST_DEBUG_OBJECT (vagg, "empty caps"); + ret = FALSE; + GST_VIDEO_AGGREGATOR_UNLOCK (vagg); + GST_OBJECT_UNLOCK (vagg); + goto done; + } + + caps = gst_caps_truncate (caps); + s = gst_caps_get_structure (caps, 0); + gst_structure_fixate_field_nearest_int (s, "width", info.width); + gst_structure_fixate_field_nearest_int (s, "height", info.height); + gst_structure_fixate_field_nearest_fraction (s, "framerate", best_fps_n, + best_fps_d); + + gst_structure_get_int (s, "width", &info.width); + gst_structure_get_int (s, "height", &info.height); + gst_structure_get_fraction (s, "framerate", &info.fps_n, &info.fps_d); + } + + gst_caps_unref (caps); + caps = gst_video_info_to_caps (&info); + + GST_VIDEO_AGGREGATOR_UNLOCK (vagg); + GST_OBJECT_UNLOCK (vagg); + + if (gst_videoaggregator_src_setcaps (vagg, caps)) { + if (vagg_klass->negotiated_caps) + ret = + GST_VIDEO_AGGREGATOR_GET_CLASS (vagg)->negotiated_caps (vagg, caps); + } + gst_caps_unref (caps); + } else { + GST_VIDEO_AGGREGATOR_UNLOCK (vagg); + } + +done: + GST_VIDEO_AGGREGATOR_SETCAPS_UNLOCK (vagg); + + return ret; +} + +static gboolean +gst_videoaggregator_pad_sink_setcaps (GstPad * pad, GstObject * parent, + GstCaps * caps) +{ + GstVideoAggregator *vagg; + GstVideoAggregatorPad *vaggpad; + GstVideoInfo info; + gboolean ret = FALSE; + + GST_INFO_OBJECT (pad, "Setting caps %" GST_PTR_FORMAT, caps); + + vagg = GST_VIDEO_AGGREGATOR (parent); + vaggpad = GST_VIDEO_AGGREGATOR_PAD (pad); + + if (!gst_video_info_from_caps (&info, caps)) { + GST_DEBUG_OBJECT (pad, "Failed to parse caps"); + goto beach; + } + + GST_VIDEO_AGGREGATOR_LOCK (vagg); + if (GST_VIDEO_INFO_FORMAT (&vagg->info) != GST_VIDEO_FORMAT_UNKNOWN) { + if (GST_VIDEO_INFO_PAR_N (&vagg->info) != GST_VIDEO_INFO_PAR_N (&info) + || GST_VIDEO_INFO_PAR_D (&vagg->info) != GST_VIDEO_INFO_PAR_D (&info) || + GST_VIDEO_INFO_INTERLACE_MODE (&vagg->info) != + GST_VIDEO_INFO_INTERLACE_MODE (&info)) { + GST_ERROR_OBJECT (pad, + "got input caps %" GST_PTR_FORMAT ", but " "current caps are %" + GST_PTR_FORMAT, caps, vagg->priv->current_caps); + GST_VIDEO_AGGREGATOR_UNLOCK (vagg); + return FALSE; + } + } + + vaggpad->info = info; + + ret = gst_videoaggregator_update_converters (vagg); + GST_VIDEO_AGGREGATOR_UNLOCK (vagg); + + if (ret) + ret = gst_videoaggregator_update_src_caps (vagg); + +beach: + return ret; +} + +static GstCaps * +gst_videoaggregator_pad_sink_getcaps (GstPad * pad, GstVideoAggregator * vagg, + GstCaps * filter) +{ + GstCaps *srccaps; + GstCaps *template_caps; + GstCaps *filtered_caps; + GstCaps *returned_caps; + GstStructure *s; + gboolean had_current_caps = TRUE; + gint i, n; + GstAggregator *agg = GST_AGGREGATOR (vagg); + + template_caps = gst_pad_get_pad_template_caps (GST_PAD (agg->srcpad)); + + srccaps = gst_pad_get_current_caps (GST_PAD (agg->srcpad)); + if (srccaps == NULL) { + had_current_caps = FALSE; + srccaps = template_caps; + } + + srccaps = gst_caps_make_writable (srccaps); + + n = gst_caps_get_size (srccaps); + for (i = 0; i < n; i++) { + s = gst_caps_get_structure (srccaps, i); + gst_structure_set (s, "width", GST_TYPE_INT_RANGE, 1, G_MAXINT, + "height", GST_TYPE_INT_RANGE, 1, G_MAXINT, + "framerate", GST_TYPE_FRACTION_RANGE, 0, 1, G_MAXINT, 1, NULL); + if (!gst_structure_has_field (s, "pixel-aspect-ratio")) + gst_structure_set (s, "pixel-aspect-ratio", GST_TYPE_FRACTION, 1, 1, + NULL); + + gst_structure_remove_fields (s, "colorimetry", "chroma-site", "format", + NULL); + } + + filtered_caps = srccaps; + if (filter) + filtered_caps = gst_caps_intersect (srccaps, filter); + returned_caps = gst_caps_intersect (filtered_caps, template_caps); + + gst_caps_unref (srccaps); + if (filter) + gst_caps_unref (filtered_caps); + if (had_current_caps) + gst_caps_unref (template_caps); + + return returned_caps; +} + +static void +gst_videoaggregator_update_qos (GstVideoAggregator * vagg, gdouble proportion, + GstClockTimeDiff diff, GstClockTime timestamp) +{ + GST_DEBUG_OBJECT (vagg, + "Updating QoS: proportion %lf, diff %s%" GST_TIME_FORMAT ", timestamp %" + GST_TIME_FORMAT, proportion, (diff < 0) ? "-" : "", + GST_TIME_ARGS (ABS (diff)), GST_TIME_ARGS (timestamp)); + + GST_OBJECT_LOCK (vagg); + vagg->priv->proportion = proportion; + if (G_LIKELY (timestamp != GST_CLOCK_TIME_NONE)) { + if (G_UNLIKELY (diff > 0)) + vagg->priv->earliest_time = + timestamp + 2 * diff + gst_util_uint64_scale_int_round (GST_SECOND, + GST_VIDEO_INFO_FPS_D (&vagg->info), + GST_VIDEO_INFO_FPS_N (&vagg->info)); + else + vagg->priv->earliest_time = timestamp + diff; + } else { + vagg->priv->earliest_time = GST_CLOCK_TIME_NONE; + } + GST_OBJECT_UNLOCK (vagg); +} + +static void +gst_videoaggregator_reset_qos (GstVideoAggregator * vagg) +{ + gst_videoaggregator_update_qos (vagg, 0.5, 0, GST_CLOCK_TIME_NONE); + vagg->priv->qos_processed = vagg->priv->qos_dropped = 0; +} + +static void +gst_videoaggregator_read_qos (GstVideoAggregator * vagg, gdouble * proportion, + GstClockTime * time) +{ + GST_OBJECT_LOCK (vagg); + *proportion = vagg->priv->proportion; + *time = vagg->priv->earliest_time; + GST_OBJECT_UNLOCK (vagg); +} + +static void +gst_videoaggregator_reset (GstVideoAggregator * vagg) +{ + GstAggregator *agg = GST_AGGREGATOR (vagg); + GList *l; + + gst_video_info_init (&vagg->info); + vagg->priv->ts_offset = 0; + vagg->priv->nframes = 0; + + gst_segment_init (&agg->segment, GST_FORMAT_TIME); + agg->segment.position = -1; + + gst_videoaggregator_reset_qos (vagg); + + GST_OBJECT_LOCK (vagg); + for (l = GST_ELEMENT (vagg)->sinkpads; l; l = l->next) { + GstVideoAggregatorPad *p = l->data; + + gst_buffer_replace (&p->buffer, NULL); + p->start_time = -1; + p->end_time = -1; + + gst_video_info_init (&p->info); + } + GST_OBJECT_UNLOCK (vagg); +} + +#define GST_FLOW_NEEDS_DATA GST_FLOW_CUSTOM_ERROR +static gint +gst_videoaggregator_fill_queues (GstVideoAggregator * vagg, + GstClockTime output_start_time, GstClockTime output_end_time) +{ + GstAggregator *agg = GST_AGGREGATOR (vagg); + GList *l; + gboolean eos = TRUE; + gboolean need_more_data = FALSE; + + GST_OBJECT_LOCK (vagg); + for (l = GST_ELEMENT (vagg)->sinkpads; l; l = l->next) { + GstVideoAggregatorPad *pad = l->data; + GstSegment *segment; + GstAggregatorPad *bpad; + GstBuffer *buf; + GstVideoInfo *vinfo; + gboolean is_eos; + + bpad = GST_AGGREGATOR_PAD (pad); + segment = &bpad->segment; + is_eos = bpad->eos; + buf = gst_aggregator_pad_get_buffer (bpad); + if (buf) { + GstClockTime start_time, end_time; + + start_time = GST_BUFFER_TIMESTAMP (buf); + if (start_time == -1) { + gst_buffer_unref (buf); + GST_DEBUG_OBJECT (pad, "Need timestamped buffers!"); + GST_OBJECT_UNLOCK (vagg); + return GST_FLOW_ERROR; + } + + vinfo = &pad->info; + + /* FIXME: Make all this work with negative rates */ + + if ((pad->buffer && start_time < GST_BUFFER_TIMESTAMP (pad->buffer)) + || (pad->queued && start_time < GST_BUFFER_TIMESTAMP (pad->queued))) { + GST_DEBUG_OBJECT (pad, "Buffer from the past, dropping"); + gst_buffer_unref (buf); + buf = gst_aggregator_pad_steal_buffer (bpad); + gst_buffer_unref (buf); + need_more_data = TRUE; + continue; + } + + if (pad->queued) { + end_time = start_time - GST_BUFFER_TIMESTAMP (pad->queued); + start_time = GST_BUFFER_TIMESTAMP (pad->queued); + gst_buffer_unref (buf); + buf = gst_buffer_ref (pad->queued); + vinfo = &pad->queued_vinfo; + } else { + end_time = GST_BUFFER_DURATION (buf); + + if (end_time == -1) { + pad->queued = buf; + buf = gst_aggregator_pad_steal_buffer (bpad); + gst_buffer_unref (buf); + pad->queued_vinfo = pad->info; + GST_DEBUG ("end time is -1 and nothing queued"); + need_more_data = TRUE; + continue; + } + } + + g_assert (start_time != -1 && end_time != -1); + end_time += start_time; /* convert from duration to position */ + + /* Check if it's inside the segment */ + if (start_time >= segment->stop || end_time < segment->start) { + GST_DEBUG_OBJECT (pad, + "Buffer outside the segment : segment: [%" GST_TIME_FORMAT " -- %" + GST_TIME_FORMAT "]" " Buffer [%" GST_TIME_FORMAT " -- %" + GST_TIME_FORMAT "]", GST_TIME_ARGS (segment->stop), + GST_TIME_ARGS (segment->start), GST_TIME_ARGS (start_time), + GST_TIME_ARGS (end_time)); + + if (buf == pad->queued) { + gst_buffer_unref (buf); + gst_buffer_replace (&pad->queued, NULL); + } else { + gst_buffer_unref (buf); + buf = gst_aggregator_pad_steal_buffer (bpad); + gst_buffer_unref (buf); + } + + need_more_data = TRUE; + continue; + } + + /* Clip to segment and convert to running time */ + start_time = MAX (start_time, segment->start); + if (segment->stop != -1) + end_time = MIN (end_time, segment->stop); + start_time = + gst_segment_to_running_time (segment, GST_FORMAT_TIME, start_time); + end_time = + gst_segment_to_running_time (segment, GST_FORMAT_TIME, end_time); + g_assert (start_time != -1 && end_time != -1); + + /* Convert to the output segment rate */ + if (ABS (agg->segment.rate) != 1.0) { + start_time *= ABS (agg->segment.rate); + end_time *= ABS (agg->segment.rate); + } + + if (pad->end_time != -1 && pad->end_time > end_time) { + GST_DEBUG_OBJECT (pad, "Buffer from the past, dropping"); + if (buf == pad->queued) { + gst_buffer_unref (buf); + gst_buffer_replace (&pad->queued, NULL); + } else { + gst_buffer_unref (buf); + buf = gst_aggregator_pad_steal_buffer (bpad); + gst_buffer_unref (buf); + } + + need_more_data = TRUE; + continue; + } + + if (end_time >= output_start_time && start_time < output_end_time) { + GST_DEBUG_OBJECT (pad, + "Taking new buffer with start time %" GST_TIME_FORMAT, + GST_TIME_ARGS (start_time)); + gst_buffer_replace (&pad->buffer, buf); + pad->buffer_vinfo = *vinfo; + pad->start_time = start_time; + pad->end_time = end_time; + + if (buf == pad->queued) { + gst_buffer_unref (buf); + gst_buffer_replace (&pad->queued, NULL); + } else { + gst_buffer_unref (buf); + buf = gst_aggregator_pad_steal_buffer (bpad); + gst_buffer_unref (buf); + } + eos = FALSE; + } else if (start_time >= output_end_time) { + GST_DEBUG_OBJECT (pad, "Keeping buffer until %" GST_TIME_FORMAT, + GST_TIME_ARGS (start_time)); + eos = FALSE; + } else { + GST_DEBUG_OBJECT (pad, "Too old buffer -- dropping"); + if (buf == pad->queued) { + gst_buffer_unref (buf); + gst_buffer_replace (&pad->queued, NULL); + } else { + gst_buffer_unref (buf); + buf = gst_aggregator_pad_steal_buffer (bpad); + gst_buffer_unref (buf); + } + + need_more_data = TRUE; + continue; + } + } else { + if (pad->end_time != -1) { + if (pad->end_time <= output_start_time) { + gst_buffer_replace (&pad->buffer, NULL); + pad->start_time = pad->end_time = -1; + if (is_eos) { + GST_DEBUG ("I just need more data"); + need_more_data = TRUE; + } + } else if (is_eos) { + eos = FALSE; + } + } + } + } + GST_OBJECT_UNLOCK (vagg); + + if (need_more_data) + return GST_FLOW_NEEDS_DATA; + if (eos) + return GST_FLOW_EOS; + + return GST_FLOW_OK; +} + +static gboolean +prepare_frames (GstVideoAggregator * vagg, GstVideoAggregatorPad * pad) +{ + GstAggregatorPad *bpad = GST_AGGREGATOR_PAD (pad); + + static GstAllocationParams params = { 0, 15, 0, 0, }; + + if (pad->buffer != NULL) { + guint outsize; + GstClockTime timestamp; + gint64 stream_time; + GstSegment *seg; + GstVideoFrame *converted_frame = g_slice_new0 (GstVideoFrame); + GstBuffer *converted_buf = NULL; + GstVideoFrame *frame = g_slice_new0 (GstVideoFrame); + + seg = &bpad->segment; + + timestamp = GST_BUFFER_TIMESTAMP (pad->buffer); + + stream_time = gst_segment_to_stream_time (seg, GST_FORMAT_TIME, timestamp); + + /* sync object properties on stream time */ + if (GST_CLOCK_TIME_IS_VALID (stream_time)) + gst_object_sync_values (GST_OBJECT (pad), stream_time); + + + if (!gst_video_frame_map (frame, &pad->buffer_vinfo, pad->buffer, + GST_MAP_READ)) { + GST_WARNING_OBJECT (vagg, "Could not map input buffer"); + } + + if (pad->convert) { + gint converted_size; + + /* We wait until here to set the conversion infos, in case vagg->info changed */ + if (pad->need_conversion_update) { + pad->conversion_info = vagg->info; + gst_video_info_set_format (&(pad->conversion_info), + GST_VIDEO_INFO_FORMAT (&vagg->info), pad->info.width, + pad->info.height); + pad->need_conversion_update = FALSE; + } + + converted_size = pad->conversion_info.size; + outsize = GST_VIDEO_INFO_SIZE (&vagg->info); + converted_size = converted_size > outsize ? converted_size : outsize; + converted_buf = gst_buffer_new_allocate (NULL, converted_size, ¶ms); + + if (!gst_video_frame_map (converted_frame, &(pad->conversion_info), + converted_buf, GST_MAP_READWRITE)) { + GST_WARNING_OBJECT (vagg, "Could not map converted frame"); + + return FALSE; + } + + videoconvert_convert_convert (pad->convert, converted_frame, frame); + pad->converted_buffer = converted_buf; + gst_video_frame_unmap (frame); + } else { + converted_frame = frame; + converted_buf = pad->buffer; + } + + pad->aggregated_frame = converted_frame; + } + + return TRUE; +} + +static gboolean +clean_pad (GstVideoAggregator * vagg, GstVideoAggregatorPad * pad) +{ + if (pad->aggregated_frame) { + gst_video_frame_unmap (pad->aggregated_frame); + g_slice_free (GstVideoFrame, pad->aggregated_frame); + pad->aggregated_frame = NULL; + } + + if (pad->converted_buffer) { + gst_buffer_unref (pad->converted_buffer); + pad->converted_buffer = NULL; + } + + return TRUE; +} + +static GstFlowReturn +gst_videoaggregator_do_aggregate (GstVideoAggregator * vagg, + GstClockTime output_start_time, GstClockTime output_end_time, + GstBuffer ** outbuf) +{ + GstFlowReturn ret = GST_FLOW_OK; + GstElementClass *klass = GST_ELEMENT_GET_CLASS (vagg); + GstVideoAggregatorClass *vagg_klass = (GstVideoAggregatorClass *) klass; + + g_assert (vagg_klass->aggregate_frames != NULL); + g_assert (vagg_klass->get_output_buffer != NULL); + + if ((ret = vagg_klass->get_output_buffer (vagg, outbuf)) != GST_FLOW_OK) { + GST_WARNING_OBJECT (vagg, "Could not get an output buffer, reason: %s", + gst_flow_get_name (ret)); + return ret; + } + + GST_BUFFER_TIMESTAMP (*outbuf) = output_start_time; + GST_BUFFER_DURATION (*outbuf) = output_end_time - output_start_time; + + if (vagg_klass->disable_frame_conversion == FALSE) { + /* Here we convert all the frames the subclass will have to aggregate */ + gst_aggregator_iterate_sinkpads (GST_AGGREGATOR (vagg), + (GstAggregatorPadForeachFunc) prepare_frames, NULL); + } + + ret = vagg_klass->aggregate_frames (vagg, *outbuf); + + gst_aggregator_iterate_sinkpads (GST_AGGREGATOR (vagg), + (GstAggregatorPadForeachFunc) clean_pad, NULL); + + return ret; +} + +/* Perform qos calculations before processing the next frame. Returns TRUE if + * the frame should be processed, FALSE if the frame can be dropped entirely */ +static gint64 +gst_videoaggregator_do_qos (GstVideoAggregator * vagg, GstClockTime timestamp) +{ + GstAggregator *agg = GST_AGGREGATOR (vagg); + GstClockTime qostime, earliest_time; + gdouble proportion; + gint64 jitter; + + /* no timestamp, can't do QoS => process frame */ + if (G_UNLIKELY (!GST_CLOCK_TIME_IS_VALID (timestamp))) { + GST_LOG_OBJECT (vagg, "invalid timestamp, can't do QoS, process frame"); + return -1; + } + + /* get latest QoS observation values */ + gst_videoaggregator_read_qos (vagg, &proportion, &earliest_time); + + /* skip qos if we have no observation (yet) => process frame */ + if (G_UNLIKELY (!GST_CLOCK_TIME_IS_VALID (earliest_time))) { + GST_LOG_OBJECT (vagg, "no observation yet, process frame"); + return -1; + } + + /* qos is done on running time */ + qostime = + gst_segment_to_running_time (&agg->segment, GST_FORMAT_TIME, timestamp); + + /* see how our next timestamp relates to the latest qos timestamp */ + GST_LOG_OBJECT (vagg, "qostime %" GST_TIME_FORMAT ", earliest %" + GST_TIME_FORMAT, GST_TIME_ARGS (qostime), GST_TIME_ARGS (earliest_time)); + + jitter = GST_CLOCK_DIFF (qostime, earliest_time); + if (qostime != GST_CLOCK_TIME_NONE && jitter > 0) { + GST_DEBUG_OBJECT (vagg, "we are late, drop frame"); + return jitter; + } + + GST_LOG_OBJECT (vagg, "process frame"); + return jitter; +} + +static GstFlowReturn +gst_videoaggregator_aggregate (GstAggregator * agg) +{ + GstFlowReturn ret; + GstVideoAggregator *vagg = GST_VIDEO_AGGREGATOR (agg); + GstClockTime output_start_time, output_end_time; + GstBuffer *outbuf = NULL; + gint res; + gint64 jitter; + + /* If we're not negotiated_caps yet... */ + if (GST_VIDEO_INFO_FORMAT (&vagg->info) == GST_VIDEO_FORMAT_UNKNOWN) { + GST_INFO_OBJECT (agg, "Not negotiated yet!"); + return GST_FLOW_NOT_NEGOTIATED; + } + + if (gst_pad_check_reconfigure (agg->srcpad)) + gst_videoaggregator_update_src_caps (vagg); + + if (vagg->priv->send_caps) { + gst_aggregator_set_src_caps (agg, vagg->priv->current_caps); + vagg->priv->send_caps = FALSE; + } + + GST_VIDEO_AGGREGATOR_LOCK (vagg); + + if (agg->segment.position == -1) + output_start_time = agg->segment.start; + else + output_start_time = agg->segment.position; + + output_end_time = + vagg->priv->ts_offset + gst_util_uint64_scale_round (vagg->priv->nframes + + 1, GST_SECOND * GST_VIDEO_INFO_FPS_D (&vagg->info), + GST_VIDEO_INFO_FPS_N (&vagg->info)) + agg->segment.start; + + if (agg->segment.stop != -1) + output_end_time = MIN (output_end_time, agg->segment.stop); + + res = + gst_videoaggregator_fill_queues (vagg, output_start_time, + output_end_time); + + if (res == GST_FLOW_NEEDS_DATA) { + GST_DEBUG_OBJECT (vagg, "Need more data for decisions"); + ret = GST_FLOW_OK; + goto done; + } else if (res == GST_FLOW_EOS) { + GST_VIDEO_AGGREGATOR_UNLOCK (vagg); + GST_DEBUG_OBJECT (vagg, "All sinkpads are EOS -- forwarding"); + ret = GST_FLOW_EOS; + goto done_unlocked; + } else if (res == GST_FLOW_ERROR) { + GST_WARNING_OBJECT (vagg, "Error collecting buffers"); + ret = GST_FLOW_ERROR; + goto done; + } + + jitter = gst_videoaggregator_do_qos (vagg, output_start_time); + if (jitter <= 0) { + ret = gst_videoaggregator_do_aggregate (vagg, output_start_time, + output_end_time, &outbuf); + vagg->priv->qos_processed++; + } else { + GstMessage *msg; + + vagg->priv->qos_dropped++; + + /* TODO: live */ + msg = + gst_message_new_qos (GST_OBJECT_CAST (vagg), FALSE, + gst_segment_to_running_time (&agg->segment, GST_FORMAT_TIME, + output_start_time), gst_segment_to_stream_time (&agg->segment, + GST_FORMAT_TIME, output_start_time), output_start_time, + output_end_time - output_start_time); + gst_message_set_qos_values (msg, jitter, vagg->priv->proportion, 1000000); + gst_message_set_qos_stats (msg, GST_FORMAT_BUFFERS, + vagg->priv->qos_processed, vagg->priv->qos_dropped); + gst_element_post_message (GST_ELEMENT_CAST (vagg), msg); + + ret = GST_FLOW_OK; + } + + agg->segment.position = output_end_time; + vagg->priv->nframes++; + + GST_VIDEO_AGGREGATOR_UNLOCK (vagg); + if (outbuf) { + GST_DEBUG_OBJECT (vagg, + "Pushing buffer with ts %" GST_TIME_FORMAT " and duration %" + GST_TIME_FORMAT, GST_TIME_ARGS (GST_BUFFER_TIMESTAMP (outbuf)), + GST_TIME_ARGS (GST_BUFFER_DURATION (outbuf))); + + ret = gst_aggregator_finish_buffer (agg, outbuf); + } + goto done_unlocked; + +done: + GST_VIDEO_AGGREGATOR_UNLOCK (vagg); + +done_unlocked: + return ret; +} + +/* FIXME, the duration query should reflect how long you will produce + * data, that is the amount of stream time until you will emit EOS. + * + * For synchronized aggregating this is always the max of all the durations + * of upstream since we emit EOS when all of them finished. + * + * We don't do synchronized aggregating so this really depends on where the + * streams where punched in and what their relative offsets are against + * each other which we can get from the first timestamps we see. + * + * When we add a new stream (or remove a stream) the duration might + * also become invalid again and we need to post a new DURATION + * message to notify this fact to the parent. + * For now we take the max of all the upstream elements so the simple + * cases work at least somewhat. + */ +static gboolean +gst_videoaggregator_query_duration (GstVideoAggregator * vagg, GstQuery * query) +{ + GValue item = { 0 }; + gint64 max; + gboolean res; + GstFormat format; + GstIterator *it; + gboolean done; + + /* parse format */ + gst_query_parse_duration (query, &format, NULL); + + max = -1; + res = TRUE; + done = FALSE; + + /* Take maximum of all durations */ + it = gst_element_iterate_sink_pads (GST_ELEMENT_CAST (vagg)); + while (!done) { + switch (gst_iterator_next (it, &item)) { + case GST_ITERATOR_DONE: + done = TRUE; + break; + case GST_ITERATOR_OK: + { + GstPad *pad; + gint64 duration; + + pad = g_value_get_object (&item); + + /* ask sink peer for duration */ + res &= gst_pad_peer_query_duration (pad, format, &duration); + /* take max from all valid return values */ + if (res) { + /* valid unknown length, stop searching */ + if (duration == -1) { + max = duration; + done = TRUE; + } + /* else see if bigger than current max */ + else if (duration > max) + max = duration; + } + g_value_reset (&item); + break; + } + case GST_ITERATOR_RESYNC: + max = -1; + res = TRUE; + gst_iterator_resync (it); + break; + default: + res = FALSE; + done = TRUE; + break; + } + } + g_value_unset (&item); + gst_iterator_free (it); + + if (res) { + /* and store the max */ + GST_DEBUG_OBJECT (vagg, "Total duration in format %s: %" + GST_TIME_FORMAT, gst_format_get_name (format), GST_TIME_ARGS (max)); + gst_query_set_duration (query, format, max); + } + + return res; +} + +static gboolean +gst_videoaggregator_query_latency (GstVideoAggregator * vagg, GstQuery * query) +{ + GstClockTime min, max; + gboolean live; + gboolean res; + GstIterator *it; + gboolean done; + GValue item = { 0 }; + + res = TRUE; + done = FALSE; + live = FALSE; + min = 0; + max = GST_CLOCK_TIME_NONE; + + /* Take maximum of all latency values */ + it = gst_element_iterate_sink_pads (GST_ELEMENT_CAST (vagg)); + while (!done) { + switch (gst_iterator_next (it, &item)) { + case GST_ITERATOR_DONE: + done = TRUE; + break; + case GST_ITERATOR_OK: + { + GstPad *pad = g_value_get_object (&item); + GstQuery *peerquery; + GstClockTime min_cur, max_cur; + gboolean live_cur; + + peerquery = gst_query_new_latency (); + + /* Ask peer for latency */ + res &= gst_pad_peer_query (pad, peerquery); + + /* take max from all valid return values */ + if (res) { + gst_query_parse_latency (peerquery, &live_cur, &min_cur, &max_cur); + + if (min_cur > min) + min = min_cur; + + if (max_cur != GST_CLOCK_TIME_NONE && + ((max != GST_CLOCK_TIME_NONE && max_cur > max) || + (max == GST_CLOCK_TIME_NONE))) + max = max_cur; + + live = live || live_cur; + } + + gst_query_unref (peerquery); + g_value_reset (&item); + break; + } + case GST_ITERATOR_RESYNC: + live = FALSE; + min = 0; + max = GST_CLOCK_TIME_NONE; + res = TRUE; + gst_iterator_resync (it); + break; + default: + res = FALSE; + done = TRUE; + break; + } + } + g_value_unset (&item); + gst_iterator_free (it); + + if (res) { + /* store the results */ + GST_DEBUG_OBJECT (vagg, "Calculated total latency: live %s, min %" + GST_TIME_FORMAT ", max %" GST_TIME_FORMAT, + (live ? "yes" : "no"), GST_TIME_ARGS (min), GST_TIME_ARGS (max)); + gst_query_set_latency (query, live, min, max); + } + + return res; +} + +static gboolean +gst_videoaggregator_src_query (GstAggregator * agg, GstQuery * query) +{ + GstVideoAggregator *vagg = GST_VIDEO_AGGREGATOR (agg); + gboolean res = FALSE; + + switch (GST_QUERY_TYPE (query)) { + case GST_QUERY_POSITION: + { + GstFormat format; + + gst_query_parse_position (query, &format, NULL); + + switch (format) { + case GST_FORMAT_TIME: + gst_query_set_position (query, format, + gst_segment_to_stream_time (&agg->segment, GST_FORMAT_TIME, + agg->segment.position)); + res = TRUE; + break; + default: + break; + } + break; + } + case GST_QUERY_DURATION: + res = gst_videoaggregator_query_duration (vagg, query); + break; + case GST_QUERY_LATENCY: + res = gst_videoaggregator_query_latency (vagg, query); + break; + case GST_QUERY_CAPS: + res = + GST_AGGREGATOR_CLASS (gst_videoaggregator_parent_class)->src_query + (agg, query); + break; + default: + /* FIXME, needs a custom query handler because we have multiple + * sinkpads */ + res = FALSE; + break; + } + return res; +} + +static gboolean +gst_videoaggregator_src_event (GstAggregator * agg, GstEvent * event) +{ + GstVideoAggregator *vagg = GST_VIDEO_AGGREGATOR (agg); + + switch (GST_EVENT_TYPE (event)) { + case GST_EVENT_QOS: + { + GstQOSType type; + GstClockTimeDiff diff; + GstClockTime timestamp; + gdouble proportion; + + gst_event_parse_qos (event, &type, &proportion, &diff, ×tamp); + gst_videoaggregator_update_qos (vagg, proportion, diff, timestamp); + break; + } + case GST_EVENT_SEEK: + { + GST_DEBUG_OBJECT (vagg, "Handling SEEK event"); + } + default: + break; + } + + return + GST_AGGREGATOR_CLASS (gst_videoaggregator_parent_class)->src_event (agg, + event); +} + +static GstFlowReturn +gst_videoaggregator_sink_clip (GstAggregator * agg, + GstAggregatorPad * bpad, GstBuffer * buf, GstBuffer ** outbuf) +{ + GstVideoAggregatorPad *pad = GST_VIDEO_AGGREGATOR_PAD (bpad); + GstClockTime start_time, end_time; + + start_time = GST_BUFFER_TIMESTAMP (buf); + if (start_time == -1) { + GST_DEBUG_OBJECT (pad, "Timestamped buffers required!"); + gst_buffer_unref (buf); + return GST_FLOW_ERROR; + } + + end_time = GST_BUFFER_DURATION (buf); + if (end_time == -1 && GST_VIDEO_INFO_FPS_N (&pad->info) != 0) + end_time = + gst_util_uint64_scale_int_round (GST_SECOND, + GST_VIDEO_INFO_FPS_D (&pad->info), GST_VIDEO_INFO_FPS_N (&pad->info)); + if (end_time == -1) { + *outbuf = buf; + return GST_FLOW_OK; + } + + start_time = MAX (start_time, bpad->segment.start); + start_time = + gst_segment_to_running_time (&bpad->segment, GST_FORMAT_TIME, start_time); + + end_time += GST_BUFFER_TIMESTAMP (buf); + if (bpad->segment.stop != -1) + end_time = MIN (end_time, bpad->segment.stop); + end_time = + gst_segment_to_running_time (&bpad->segment, GST_FORMAT_TIME, end_time); + + /* Convert to the output segment rate */ + if (ABS (agg->segment.rate) != 1.0) { + start_time *= ABS (agg->segment.rate); + end_time *= ABS (agg->segment.rate); + } + + if (bpad->buffer != NULL && end_time < pad->end_time) { + gst_buffer_unref (buf); + *outbuf = NULL; + return GST_FLOW_OK; + } + + *outbuf = buf; + return GST_FLOW_OK; +} + +static GstFlowReturn +gst_videoaggregator_flush (GstAggregator * agg) +{ + GList *l; + gdouble abs_rate; + GstVideoAggregator *vagg = GST_VIDEO_AGGREGATOR (agg); + + GST_INFO_OBJECT (agg, "Flushing"); + abs_rate = ABS (agg->segment.rate); + GST_OBJECT_LOCK (vagg); + for (l = GST_ELEMENT (vagg)->sinkpads; l; l = l->next) { + GstVideoAggregatorPad *p = l->data; + + /* Convert to the output segment rate */ + if (ABS (agg->segment.rate) != abs_rate) { + if (ABS (agg->segment.rate) != 1.0 && p->buffer) { + p->start_time /= ABS (agg->segment.rate); + p->end_time /= ABS (agg->segment.rate); + } + if (abs_rate != 1.0 && p->buffer) { + p->start_time *= abs_rate; + p->end_time *= abs_rate; + } + } + } + GST_OBJECT_UNLOCK (vagg); + + agg->segment.position = -1; + vagg->priv->ts_offset = 0; + vagg->priv->nframes = 0; + + gst_videoaggregator_reset_qos (vagg); + return GST_FLOW_OK; +} + +static gboolean +gst_videoaggregator_sink_event (GstAggregator * agg, GstAggregatorPad * bpad, + GstEvent * event) +{ + GstVideoAggregator *vagg = GST_VIDEO_AGGREGATOR (agg); + GstVideoAggregatorPad *pad = GST_VIDEO_AGGREGATOR_PAD (bpad); + gboolean ret = TRUE; + + GST_DEBUG_OBJECT (pad, "Got %s event on pad %s:%s", + GST_EVENT_TYPE_NAME (event), GST_DEBUG_PAD_NAME (pad)); + + switch (GST_EVENT_TYPE (event)) { + case GST_EVENT_CAPS: + { + GstCaps *caps; + + gst_event_parse_caps (event, &caps); + ret = + gst_videoaggregator_pad_sink_setcaps (GST_PAD (pad), + GST_OBJECT (vagg), caps); + gst_event_unref (event); + event = NULL; + break; + } + case GST_EVENT_SEGMENT:{ + GstSegment seg; + gst_event_copy_segment (event, &seg); + + g_assert (seg.format == GST_FORMAT_TIME); + break; + } + default: + break; + } + + if (event != NULL) + return GST_AGGREGATOR_CLASS (gst_videoaggregator_parent_class)->sink_event + (agg, bpad, event); + + return ret; +} + +static gboolean +gst_videoaggregator_start (GstAggregator * agg) +{ + GstVideoAggregator *vagg = GST_VIDEO_AGGREGATOR (agg); + + if (!GST_AGGREGATOR_CLASS (gst_videoaggregator_parent_class)->start (agg)) + return FALSE; + + vagg->priv->send_caps = TRUE; + gst_segment_init (&agg->segment, GST_FORMAT_TIME); + gst_caps_replace (&vagg->priv->current_caps, NULL); + + return TRUE; +} + +static gboolean +gst_videoaggregator_stop (GstAggregator * agg) +{ + GstVideoAggregator *vagg = GST_VIDEO_AGGREGATOR (agg); + + if (!GST_AGGREGATOR_CLASS (gst_videoaggregator_parent_class)->stop (agg)) + return FALSE; + + gst_videoaggregator_reset (vagg); + + return TRUE; +} + +/* GstElement vmethods */ +static GstPad * +gst_videoaggregator_request_new_pad (GstElement * element, + GstPadTemplate * templ, const gchar * req_name, const GstCaps * caps) +{ + GstVideoAggregator *vagg; + GstVideoAggregatorPad *vaggpad; + + vagg = GST_VIDEO_AGGREGATOR (element); + + vaggpad = (GstVideoAggregatorPad *) + GST_ELEMENT_CLASS (gst_videoaggregator_parent_class)->request_new_pad + (element, templ, req_name, caps); + + if (vaggpad == NULL) + return NULL; + + GST_OBJECT_LOCK (vagg); + vaggpad->zorder = GST_ELEMENT (vagg)->numsinkpads; + vaggpad->start_time = -1; + vaggpad->end_time = -1; + element->sinkpads = g_list_sort (element->sinkpads, + (GCompareFunc) pad_zorder_compare); + GST_OBJECT_UNLOCK (vagg); + + gst_child_proxy_child_added (GST_CHILD_PROXY (vagg), G_OBJECT (vaggpad), + GST_OBJECT_NAME (vaggpad)); + + return GST_PAD (vaggpad); +} + +static void +gst_videoaggregator_release_pad (GstElement * element, GstPad * pad) +{ + GstVideoAggregator *vagg = NULL; + GstVideoAggregatorPad *vaggpad; + gboolean update_caps; + + vagg = GST_VIDEO_AGGREGATOR (element); + vaggpad = GST_VIDEO_AGGREGATOR_PAD (pad); + + GST_VIDEO_AGGREGATOR_LOCK (vagg); + gst_videoaggregator_update_converters (vagg); + gst_buffer_replace (&vaggpad->buffer, NULL); + update_caps = GST_VIDEO_INFO_FORMAT (&vagg->info) != GST_VIDEO_FORMAT_UNKNOWN; + GST_VIDEO_AGGREGATOR_UNLOCK (vagg); + + gst_child_proxy_child_removed (GST_CHILD_PROXY (vagg), G_OBJECT (vaggpad), + GST_OBJECT_NAME (vaggpad)); + + GST_ELEMENT_CLASS (gst_videoaggregator_parent_class)->release_pad (GST_ELEMENT + (vagg), pad); + + if (update_caps) + gst_videoaggregator_update_src_caps (vagg); + + return; +} + +static GstFlowReturn +gst_videoaggregator_get_output_buffer (GstVideoAggregator * videoaggregator, + GstBuffer ** outbuf) +{ + guint outsize; + static GstAllocationParams params = { 0, 15, 0, 0, }; + + outsize = GST_VIDEO_INFO_SIZE (&videoaggregator->info); + *outbuf = gst_buffer_new_allocate (NULL, outsize, ¶ms); + + if (*outbuf == NULL) { + GST_ERROR_OBJECT (videoaggregator, + "Could not instantiate buffer of size: %d", outsize); + } + + return GST_FLOW_OK; +} + +static gboolean +gst_videoaggregator_pad_sink_acceptcaps (GstPad * pad, + GstVideoAggregator * vagg, GstCaps * caps) +{ + gboolean ret; + GstCaps *modified_caps; + GstCaps *accepted_caps; + GstCaps *template_caps; + gboolean had_current_caps = TRUE; + gint i, n; + GstStructure *s; + GstAggregator *agg = GST_AGGREGATOR (vagg); + + GST_DEBUG_OBJECT (pad, "%" GST_PTR_FORMAT, caps); + + accepted_caps = gst_pad_get_current_caps (GST_PAD (agg->srcpad)); + + template_caps = gst_pad_get_pad_template_caps (GST_PAD (agg->srcpad)); + + if (accepted_caps == NULL) { + accepted_caps = template_caps; + had_current_caps = FALSE; + } + + accepted_caps = gst_caps_make_writable (accepted_caps); + + GST_LOG_OBJECT (pad, "src caps %" GST_PTR_FORMAT, accepted_caps); + + n = gst_caps_get_size (accepted_caps); + for (i = 0; i < n; i++) { + s = gst_caps_get_structure (accepted_caps, i); + gst_structure_set (s, "width", GST_TYPE_INT_RANGE, 1, G_MAXINT, + "height", GST_TYPE_INT_RANGE, 1, G_MAXINT, + "framerate", GST_TYPE_FRACTION_RANGE, 0, 1, G_MAXINT, 1, NULL); + if (!gst_structure_has_field (s, "pixel-aspect-ratio")) + gst_structure_set (s, "pixel-aspect-ratio", GST_TYPE_FRACTION, 1, 1, + NULL); + + gst_structure_remove_fields (s, "colorimetry", "chroma-site", "format", + NULL); + } + + modified_caps = gst_caps_intersect (accepted_caps, template_caps); + + ret = gst_caps_can_intersect (caps, accepted_caps); + GST_DEBUG_OBJECT (pad, "%saccepted caps %" GST_PTR_FORMAT, + (ret ? "" : "not "), caps); + gst_caps_unref (accepted_caps); + gst_caps_unref (modified_caps); + if (had_current_caps) + gst_caps_unref (template_caps); + return ret; +} + +static gboolean +gst_videoaggregator_sink_query (GstAggregator * agg, GstAggregatorPad * bpad, + GstQuery * query) +{ + GstVideoAggregator *vagg = GST_VIDEO_AGGREGATOR (agg); + GstVideoAggregatorPad *pad = GST_VIDEO_AGGREGATOR_PAD (bpad); + gboolean ret = FALSE; + + switch (GST_QUERY_TYPE (query)) { + case GST_QUERY_CAPS: + { + GstCaps *filter, *caps; + + gst_query_parse_caps (query, &filter); + caps = gst_videoaggregator_pad_sink_getcaps (GST_PAD (pad), vagg, filter); + gst_query_set_caps_result (query, caps); + gst_caps_unref (caps); + ret = TRUE; + break; + } + case GST_QUERY_ACCEPT_CAPS: + { + GstCaps *caps; + + gst_query_parse_accept_caps (query, &caps); + ret = gst_videoaggregator_pad_sink_acceptcaps (GST_PAD (pad), vagg, caps); + gst_query_set_accept_caps_result (query, ret); + ret = TRUE; + break; + } + default: + ret = + GST_AGGREGATOR_CLASS (gst_videoaggregator_parent_class)->sink_query + (agg, bpad, query); + break; + } + return ret; +} + +/* GObject vmethods */ +static void +gst_videoaggregator_finalize (GObject * o) +{ + GstVideoAggregator *vagg = GST_VIDEO_AGGREGATOR (o); + + g_mutex_clear (&vagg->priv->lock); + g_mutex_clear (&vagg->priv->setcaps_lock); + + G_OBJECT_CLASS (gst_videoaggregator_parent_class)->finalize (o); +} + +static void +gst_videoaggregator_dispose (GObject * o) +{ + GstVideoAggregator *vagg = GST_VIDEO_AGGREGATOR (o); + + gst_caps_replace (&vagg->priv->current_caps, NULL); +} + +static void +gst_videoaggregator_get_property (GObject * object, + guint prop_id, GValue * value, GParamSpec * pspec) +{ + switch (prop_id) { + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static void +gst_videoaggregator_set_property (GObject * object, + guint prop_id, const GValue * value, GParamSpec * pspec) +{ + switch (prop_id) { + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +/* GObject boilerplate */ +static void +gst_videoaggregator_class_init (GstVideoAggregatorClass * klass) +{ + GObjectClass *gobject_class = (GObjectClass *) klass; + GstElementClass *gstelement_class = (GstElementClass *) klass; + GstAggregatorClass *agg_class = (GstAggregatorClass *) klass; + + GST_DEBUG_CATEGORY_INIT (gst_videoaggregator_debug, "videoaggregator", 0, + "base video aggregator"); + + g_type_class_add_private (klass, sizeof (GstVideoAggregatorPrivate)); + + gobject_class->finalize = gst_videoaggregator_finalize; + gobject_class->dispose = gst_videoaggregator_dispose; + + gobject_class->get_property = gst_videoaggregator_get_property; + gobject_class->set_property = gst_videoaggregator_set_property; + + gstelement_class->request_new_pad = + GST_DEBUG_FUNCPTR (gst_videoaggregator_request_new_pad); + gstelement_class->release_pad = + GST_DEBUG_FUNCPTR (gst_videoaggregator_release_pad); + + gst_element_class_set_static_metadata (gstelement_class, + "Video aggregator base class", "Filter/Editor/Video", + "Aggregate multiple video streams", + "Wim Taymans , " + "Sebastian Dröge , " + "Mathieu Duponchelle , " + "Thibault Saunier "); + + agg_class->sinkpads_type = GST_TYPE_VIDEO_AGGREGATOR_PAD; + agg_class->start = gst_videoaggregator_start; + agg_class->stop = gst_videoaggregator_stop; + agg_class->sink_query = gst_videoaggregator_sink_query; + agg_class->sink_event = gst_videoaggregator_sink_event; + agg_class->flush = gst_videoaggregator_flush; + agg_class->clip = gst_videoaggregator_sink_clip; + agg_class->aggregate = gst_videoaggregator_aggregate; + agg_class->src_event = gst_videoaggregator_src_event; + agg_class->src_query = gst_videoaggregator_src_query; + + klass->get_output_buffer = gst_videoaggregator_get_output_buffer; + + /* Register the pad class */ + g_type_class_ref (GST_TYPE_VIDEO_AGGREGATOR_PAD); +} + +static void +gst_videoaggregator_init (GstVideoAggregator * vagg) +{ + vagg->priv = + G_TYPE_INSTANCE_GET_PRIVATE (vagg, GST_TYPE_VIDEO_AGGREGATOR, + GstVideoAggregatorPrivate); + + vagg->priv->current_caps = NULL; + + g_mutex_init (&vagg->priv->lock); + g_mutex_init (&vagg->priv->setcaps_lock); + /* initialize variables */ + gst_videoaggregator_reset (vagg); +} diff --git a/gst-libs/gst/video/gstvideoaggregator.h b/gst-libs/gst/video/gstvideoaggregator.h new file mode 100644 index 0000000000..9fa5071ae6 --- /dev/null +++ b/gst-libs/gst/video/gstvideoaggregator.h @@ -0,0 +1,109 @@ +/* Generic video aggregator plugin + * Copyright (C) 2008 Wim Taymans + * Copyright (C) 2010 Sebastian Dröge + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifndef __GST_VIDEO_AGGREGATOR_H__ +#define __GST_VIDEO_AGGREGATOR_H__ + +#include +#include +#include + +#include "gstvideoaggregatorpad.h" + +G_BEGIN_DECLS + +#define GST_TYPE_VIDEO_AGGREGATOR (gst_videoaggregator_get_type()) +#define GST_VIDEO_AGGREGATOR(obj) \ + (G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_VIDEO_AGGREGATOR, GstVideoAggregator)) +#define GST_VIDEO_AGGREGATOR_CLASS(klass) \ + (G_TYPE_CHECK_CLASS_CAST((klass),GST_TYPE_VIDEO_AGGREGATOR, GstVideoAggregatorClass)) +#define GST_IS_VIDEO_AGGREGATOR(obj) \ + (G_TYPE_CHECK_INSTANCE_TYPE((obj),GST_TYPE_VIDEO_AGGREGATOR)) +#define GST_IS_VIDEO_AGGREGATOR_CLASS(klass) \ + (G_TYPE_CHECK_CLASS_TYPE((klass),GST_TYPE_VIDEO_AGGREGATOR)) +#define GST_VIDEO_AGGREGATOR_GET_CLASS(obj) \ + (G_TYPE_INSTANCE_GET_CLASS((obj),GST_TYPE_VIDEO_AGGREGATOR,GstVideoAggregatorClass)) + +typedef struct _GstVideoAggregator GstVideoAggregator; +typedef struct _GstVideoAggregatorClass GstVideoAggregatorClass; +typedef struct _GstVideoAggregatorPrivate GstVideoAggregatorPrivate; + +/** + * GstVideoAggregator: + * @info: The #GstVideoInfo representing the currently set + * srcpad caps. + */ +struct _GstVideoAggregator +{ + GstAggregator aggregator; + + /*< public >*/ + /* Output caps */ + GstVideoInfo info; + + /* < private > */ + GstVideoAggregatorPrivate *priv; + gpointer _gst_reserved[GST_PADDING]; +}; + +/** + * GstVideoAggregatorClass: + * @disable_frame_conversion: Optional. + * Allows subclasses to disable the frame colorspace + * conversion feature + * @update_info: Optional. + * Lets subclasses update the src #GstVideoInfo representing + * the src pad caps before usage. + * @aggregate_frames: Lets subclasses aggregate frames that are ready. Subclasses + * should iterate the GstElement.sinkpads and use the already + * mapped #GstVideoFrame from GstVideoAggregatorPad.aggregated_frame + * or directly use the #GstBuffer from GstVideoAggregatorPad.buffer + * if it needs to map the buffer in a special way. The result of the + * aggregation should land in @outbuffer. + * @get_output_buffer: Optional. + * Lets subclasses provide a #GstBuffer to be used as @outbuffer of + * the #aggregate_frames vmethod. + * @negotiated_caps: Optional. + * Notifies subclasses what caps format has been negotiated + **/ +struct _GstVideoAggregatorClass +{ + /*< private >*/ + GstAggregatorClass parent_class; + + /*< public >*/ + gboolean disable_frame_conversion; + + gboolean (*update_info) (GstVideoAggregator * videoaggregator, + GstVideoInfo * info); + GstFlowReturn (*aggregate_frames) (GstVideoAggregator * videoaggregator, + GstBuffer * outbuffer); + GstFlowReturn (*get_output_buffer) (GstVideoAggregator * videoaggregator, + GstBuffer ** outbuffer); + gboolean (*negotiated_caps) (GstVideoAggregator * videoaggregator, + GstCaps * caps); + /* < private > */ + gpointer _gst_reserved[GST_PADDING]; +}; + +GType gst_videoaggregator_get_type (void); + +G_END_DECLS +#endif /* __GST_VIDEO_AGGREGATOR_H__ */ From 59a7205ae0263a580417547dd0ceda79eabe3a3c Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Tue, 10 Jun 2014 11:26:53 +0200 Subject: [PATCH 010/381] gl: Port glmixer to the GstVideoAggregator baseclass https://bugzilla.gnome.org/show_bug.cgi?id=731921 --- ext/gl/gstglmosaic.c | 11 +++++----- ext/gl/gstglvideomixer.c | 47 +++++++++++----------------------------- 2 files changed, 19 insertions(+), 39 deletions(-) diff --git a/ext/gl/gstglmosaic.c b/ext/gl/gstglmosaic.c index 857c179931..5a5c41835e 100644 --- a/ext/gl/gstglmosaic.c +++ b/ext/gl/gstglmosaic.c @@ -197,9 +197,9 @@ gst_gl_mosaic_process_textures (GstGLMixer * mix, GPtrArray * frames, //blocking call, use a FBO gst_gl_context_use_fbo_v2 (mix->context, - GST_VIDEO_INFO_WIDTH (&mix->out_info), - GST_VIDEO_INFO_HEIGHT (&mix->out_info), mix->fbo, mix->depthbuffer, - out_tex, gst_gl_mosaic_callback, (gpointer) mosaic); + GST_VIDEO_INFO_WIDTH (&GST_VIDEO_AGGREGATOR (mix)->info), + GST_VIDEO_INFO_HEIGHT (&GST_VIDEO_AGGREGATOR (mix)->info), mix->fbo, + mix->depthbuffer, out_tex, gst_gl_mosaic_callback, (gpointer) mosaic); return TRUE; } @@ -294,8 +294,9 @@ gst_gl_mosaic_callback (gpointer stuff) continue; } in_tex = frame->texture; - width = GST_VIDEO_INFO_WIDTH (&frame->pad->in_info); - height = GST_VIDEO_INFO_HEIGHT (&frame->pad->in_info); + width = GST_VIDEO_INFO_WIDTH (&GST_VIDEO_AGGREGATOR_PAD (frame->pad)->info); + height = + GST_VIDEO_INFO_HEIGHT (&GST_VIDEO_AGGREGATOR_PAD (frame->pad)->info); if (!in_tex || width <= 0 || height <= 0) { GST_DEBUG ("skipping texture:%u frame:%p width:%u height %u", diff --git a/ext/gl/gstglvideomixer.c b/ext/gl/gstglvideomixer.c index e3d5807410..b2ff4008f9 100644 --- a/ext/gl/gstglvideomixer.c +++ b/ext/gl/gstglvideomixer.c @@ -116,7 +116,6 @@ struct _GstGLVideoMixerPad /* properties */ gint xpos, ypos; gint width, height; - guint zorder; gdouble alpha; }; @@ -134,7 +133,6 @@ static void gst_gl_video_mixer_pad_set_property (GObject * object, static void gst_gl_video_mixer_pad_get_property (GObject * object, guint prop_id, GValue * value, GParamSpec * pspec); -#define DEFAULT_PAD_ZORDER 0 #define DEFAULT_PAD_XPOS 0 #define DEFAULT_PAD_YPOS 0 #define DEFAULT_PAD_WIDTH 0 @@ -143,7 +141,6 @@ static void gst_gl_video_mixer_pad_get_property (GObject * object, enum { PROP_PAD_0, - PROP_PAD_ZORDER, PROP_PAD_XPOS, PROP_PAD_YPOS, PROP_PAD_WIDTH, @@ -165,10 +162,6 @@ gst_gl_video_mixer_pad_class_init (GstGLVideoMixerPadClass * klass) gobject_class->set_property = gst_gl_video_mixer_pad_set_property; gobject_class->get_property = gst_gl_video_mixer_pad_get_property; - g_object_class_install_property (gobject_class, PROP_PAD_ZORDER, - g_param_spec_uint ("zorder", "Z-Order", "Z Order of the picture", - 0, 10000, DEFAULT_PAD_ZORDER, - G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); g_object_class_install_property (gobject_class, PROP_PAD_XPOS, g_param_spec_int ("xpos", "X Position", "X Position of the picture", G_MININT, G_MAXINT, DEFAULT_PAD_XPOS, @@ -198,9 +191,6 @@ gst_gl_video_mixer_pad_get_property (GObject * object, guint prop_id, GstGLVideoMixerPad *pad = GST_GL_VIDEO_MIXER_PAD (object); switch (prop_id) { - case PROP_PAD_ZORDER: - g_value_set_uint (value, pad->zorder); - break; case PROP_PAD_XPOS: g_value_set_int (value, pad->xpos); break; @@ -222,13 +212,6 @@ gst_gl_video_mixer_pad_get_property (GObject * object, guint prop_id, } } -static int -pad_zorder_compare (const GstGLVideoMixerPad * pad1, - const GstGLVideoMixerPad * pad2) -{ - return pad1->zorder - pad2->zorder; -} - static void gst_gl_video_mixer_pad_set_property (GObject * object, guint prop_id, const GValue * value, GParamSpec * pspec) @@ -237,14 +220,6 @@ gst_gl_video_mixer_pad_set_property (GObject * object, guint prop_id, GstGLMixer *mix = GST_GL_MIXER (gst_pad_get_parent (GST_PAD (pad))); switch (prop_id) { - case PROP_PAD_ZORDER: - GST_GL_MIXER_LOCK (mix); - pad->zorder = g_value_get_uint (value); - - mix->sinkpads = g_slist_sort (mix->sinkpads, - (GCompareFunc) pad_zorder_compare); - GST_GL_MIXER_UNLOCK (mix); - break; case PROP_PAD_XPOS: pad->xpos = g_value_get_int (value); break; @@ -273,6 +248,7 @@ gst_gl_video_mixer_class_init (GstGLVideoMixerClass * klass) { GObjectClass *gobject_class; GstElementClass *element_class; + GstAggregatorClass *agg_class = (GstAggregatorClass *) klass; gobject_class = (GObjectClass *) klass; element_class = GST_ELEMENT_CLASS (klass); @@ -288,6 +264,9 @@ gst_gl_video_mixer_class_init (GstGLVideoMixerClass * klass) GST_GL_MIXER_CLASS (klass)->reset = gst_gl_video_mixer_reset; GST_GL_MIXER_CLASS (klass)->process_textures = gst_gl_video_mixer_process_textures; + + agg_class->sinkpads_type = GST_TYPE_GL_VIDEO_MIXER_PAD; + } static void @@ -295,9 +274,6 @@ gst_gl_video_mixer_init (GstGLVideoMixer * video_mixer) { video_mixer->shader = NULL; video_mixer->input_frames = NULL; - - gst_gl_mixer_set_pad_type (GST_GL_MIXER (video_mixer), - GST_TYPE_GL_VIDEO_MIXER_PAD); } static void @@ -352,8 +328,9 @@ gst_gl_video_mixer_process_textures (GstGLMixer * mix, GPtrArray * frames, video_mixer->input_frames = frames; gst_gl_context_use_fbo_v2 (mix->context, - GST_VIDEO_INFO_WIDTH (&mix->out_info), - GST_VIDEO_INFO_HEIGHT (&mix->out_info), mix->fbo, mix->depthbuffer, + GST_VIDEO_INFO_WIDTH (&GST_VIDEO_AGGREGATOR (mix)->info), + GST_VIDEO_INFO_HEIGHT (&GST_VIDEO_AGGREGATOR (mix)->info), + mix->fbo, mix->depthbuffer, out_tex, gst_gl_video_mixer_callback, (gpointer) video_mixer); return TRUE; @@ -378,8 +355,8 @@ gst_gl_video_mixer_callback (gpointer stuff) guint count = 0; - out_width = GST_VIDEO_INFO_WIDTH (&mixer->out_info); - out_height = GST_VIDEO_INFO_HEIGHT (&mixer->out_info); + out_width = GST_VIDEO_INFO_WIDTH (&GST_VIDEO_AGGREGATOR (stuff)->info); + out_height = GST_VIDEO_INFO_HEIGHT (&GST_VIDEO_AGGREGATOR (stuff)->info); gst_gl_context_clear_shader (mixer->context); gl->BindTexture (GL_TEXTURE_2D, 0); @@ -424,8 +401,10 @@ gst_gl_video_mixer_callback (gpointer stuff) continue; } pad = (GstGLVideoMixerPad *) frame->pad; - in_width = GST_VIDEO_INFO_WIDTH (&frame->pad->in_info); - in_height = GST_VIDEO_INFO_HEIGHT (&frame->pad->in_info); + in_width = + GST_VIDEO_INFO_WIDTH (&GST_VIDEO_AGGREGATOR_PAD (pad)->info); + in_height = + GST_VIDEO_INFO_HEIGHT (&GST_VIDEO_AGGREGATOR_PAD (pad)->info); if (!frame->texture || in_width <= 0 || in_height <= 0) { GST_DEBUG ("skipping texture:%u frame:%p width:%u height %u", From 75c71741a21163fdaa2b21064323916de29d658d Mon Sep 17 00:00:00 2001 From: Mathieu Duponchelle Date: Thu, 22 May 2014 19:46:02 +0200 Subject: [PATCH 011/381] compositor: Add a new compositor based on the new GstVideoAggregator base class It is a replacement for videomixer with a similare API Co-Authored by: Thibault Saunier https://bugzilla.gnome.org/show_bug.cgi?id=731919 --- gst/compositor/Makefile.am | 29 + gst/compositor/blend.c | 1059 ++++++++++++ gst/compositor/blend.h | 103 ++ gst/compositor/blendorc.h | 96 ++ gst/compositor/compositor.c | 604 +++++++ gst/compositor/compositor.h | 86 + gst/compositor/compositororc-dist.c | 2400 +++++++++++++++++++++++++++ gst/compositor/compositororc-dist.h | 96 ++ gst/compositor/compositororc.orc | 220 +++ gst/compositor/compositorpad.h | 65 + tests/check/elements/compositor.c | 1071 ++++++++++++ 11 files changed, 5829 insertions(+) create mode 100644 gst/compositor/Makefile.am create mode 100644 gst/compositor/blend.c create mode 100644 gst/compositor/blend.h create mode 100644 gst/compositor/blendorc.h create mode 100644 gst/compositor/compositor.c create mode 100644 gst/compositor/compositor.h create mode 100644 gst/compositor/compositororc-dist.c create mode 100644 gst/compositor/compositororc-dist.h create mode 100644 gst/compositor/compositororc.orc create mode 100644 gst/compositor/compositorpad.h create mode 100644 tests/check/elements/compositor.c diff --git a/gst/compositor/Makefile.am b/gst/compositor/Makefile.am new file mode 100644 index 0000000000..a91f0fa4f2 --- /dev/null +++ b/gst/compositor/Makefile.am @@ -0,0 +1,29 @@ +plugin_LTLIBRARIES = libgstcompositor.la + +ORC_SOURCE=compositororc + +include $(top_srcdir)/common/orc.mak + +libgstcompositor_la_SOURCES = \ + blend.c \ + compositor.c + + +nodist_libgstcompositor_la_SOURCES = $(ORC_NODIST_SOURCES) +libgstcompositor_la_CFLAGS = $(GST_PLUGINS_BASE_CFLAGS) \ + -I$(top_srcdir)/gst-libs \ + -I$(top_builddir)/gst-libs \ + $(GST_BASE_CFLAGS) $(GST_CFLAGS) $(ORC_CFLAGS) +libgstcompositor_la_LIBADD = $(GST_PLUGINS_BASE_LIBS) \ + -lgstvideo-@GST_API_VERSION@ \ + $(top_builddir)/gst-libs/gst/base/libgstbadbase-$(GST_API_VERSION).la \ + $(top_builddir)/gst-libs/gst/video/libgstbadvideo-$(GST_API_VERSION).la \ + $(GST_BASE_LIBS) $(GST_LIBS) $(ORC_LIBS) $(LIBM) +libgstcompositor_la_LDFLAGS = $(GST_PLUGIN_LDFLAGS) +libgstcompositor_la_LIBTOOLFLAGS = $(GST_PLUGIN_LIBTOOLFLAGS) + +# headers we need but don't want installed +noinst_HEADERS = \ + blend.h \ + compositor.h \ + compositorpad.h diff --git a/gst/compositor/blend.c b/gst/compositor/blend.c new file mode 100644 index 0000000000..7fe8e53fd8 --- /dev/null +++ b/gst/compositor/blend.c @@ -0,0 +1,1059 @@ +/* + * Copyright (C) 2004 Wim Taymans + * Copyright (C) 2006 Mindfruit Bv. + * Author: Sjoerd Simons + * Author: Alex Ugarte + * Copyright (C) 2009 Alex Ugarte + * Copyright (C) 2009 Sebastian Dröge + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "blend.h" +#include "compositororc.h" + +#include + +#include + +#define BLEND(D,S,alpha) (((D) * (256 - (alpha)) + (S) * (alpha)) >> 8) + +GST_DEBUG_CATEGORY_STATIC (gst_compositor_blend_debug); +#define GST_CAT_DEFAULT gst_compositor_blend_debug + +/* Below are the implementations of everything */ + +/* A32 is for AYUV, ARGB and BGRA */ +#define BLEND_A32(name, method, LOOP) \ +static void \ +method##_ ##name (GstVideoFrame * srcframe, gint xpos, gint ypos, \ + gdouble src_alpha, GstVideoFrame * destframe) \ +{ \ + guint s_alpha; \ + gint src_stride, dest_stride; \ + gint dest_width, dest_height; \ + guint8 *src, *dest; \ + gint src_width, src_height; \ + \ + src_width = GST_VIDEO_FRAME_WIDTH (srcframe); \ + src_height = GST_VIDEO_FRAME_HEIGHT (srcframe); \ + src = GST_VIDEO_FRAME_PLANE_DATA (srcframe, 0); \ + src_stride = GST_VIDEO_FRAME_COMP_STRIDE (srcframe, 0); \ + dest = GST_VIDEO_FRAME_PLANE_DATA (destframe, 0); \ + dest_stride = GST_VIDEO_FRAME_COMP_STRIDE (destframe, 0); \ + dest_width = GST_VIDEO_FRAME_COMP_WIDTH (destframe, 0); \ + dest_height = GST_VIDEO_FRAME_COMP_HEIGHT (destframe, 0); \ + \ + s_alpha = CLAMP ((gint) (src_alpha * 256), 0, 256); \ + \ + /* If it's completely transparent... we just return */ \ + if (G_UNLIKELY (s_alpha == 0)) \ + return; \ + \ + /* adjust src pointers for negative sizes */ \ + if (xpos < 0) { \ + src += -xpos * 4; \ + src_width -= -xpos; \ + xpos = 0; \ + } \ + if (ypos < 0) { \ + src += -ypos * src_stride; \ + src_height -= -ypos; \ + ypos = 0; \ + } \ + /* adjust width/height if the src is bigger than dest */ \ + if (xpos + src_width > dest_width) { \ + src_width = dest_width - xpos; \ + } \ + if (ypos + src_height > dest_height) { \ + src_height = dest_height - ypos; \ + } \ + \ + dest = dest + 4 * xpos + (ypos * dest_stride); \ + \ + LOOP (dest, src, src_height, src_width, src_stride, dest_stride, s_alpha); \ +} + +#define BLEND_A32_LOOP(name, method) \ +static inline void \ +_##method##_loop_##name (guint8 * dest, const guint8 * src, gint src_height, \ + gint src_width, gint src_stride, gint dest_stride, guint s_alpha) \ +{ \ + s_alpha = MIN (255, s_alpha); \ + compositor_orc_##method##_##name (dest, dest_stride, src, src_stride, \ + s_alpha, src_width, src_height); \ +} + +BLEND_A32_LOOP (argb, blend); +BLEND_A32_LOOP (bgra, blend); +BLEND_A32_LOOP (argb, overlay); +BLEND_A32_LOOP (bgra, overlay); + +#if G_BYTE_ORDER == G_LITTLE_ENDIAN +BLEND_A32 (argb, blend, _blend_loop_argb); +BLEND_A32 (bgra, blend, _blend_loop_bgra); +BLEND_A32 (argb, overlay, _overlay_loop_argb); +BLEND_A32 (bgra, overlay, _overlay_loop_bgra); +#else +BLEND_A32 (argb, blend, _blend_loop_bgra); +BLEND_A32 (bgra, blend, _blend_loop_argb); +BLEND_A32 (argb, overlay, _overlay_loop_bgra); +BLEND_A32 (bgra, overlay, _overlay_loop_argb); +#endif + +#define A32_CHECKER_C(name, RGB, A, C1, C2, C3) \ +static void \ +fill_checker_##name##_c (GstVideoFrame * frame) \ +{ \ + gint i, j; \ + gint val; \ + static const gint tab[] = { 80, 160, 80, 160 }; \ + gint width, height; \ + guint8 *dest; \ + \ + dest = GST_VIDEO_FRAME_PLANE_DATA (frame, 0); \ + width = GST_VIDEO_FRAME_COMP_WIDTH (frame, 0); \ + height = GST_VIDEO_FRAME_COMP_HEIGHT (frame, 0); \ + \ + if (!RGB) { \ + for (i = 0; i < height; i++) { \ + for (j = 0; j < width; j++) { \ + dest[A] = 0xff; \ + dest[C1] = tab[((i & 0x8) >> 3) + ((j & 0x8) >> 3)]; \ + dest[C2] = 128; \ + dest[C3] = 128; \ + dest += 4; \ + } \ + } \ + } else { \ + for (i = 0; i < height; i++) { \ + for (j = 0; j < width; j++) { \ + val = tab[((i & 0x8) >> 3) + ((j & 0x8) >> 3)]; \ + dest[A] = 0xFF; \ + dest[C1] = val; \ + dest[C2] = val; \ + dest[C3] = val; \ + dest += 4; \ + } \ + } \ + } \ +} + +A32_CHECKER_C (argb, TRUE, 0, 1, 2, 3); +A32_CHECKER_C (bgra, TRUE, 3, 2, 1, 0); +A32_CHECKER_C (ayuv, FALSE, 0, 1, 2, 3); + +#define YUV_TO_R(Y,U,V) (CLAMP (1.164 * (Y - 16) + 1.596 * (V - 128), 0, 255)) +#define YUV_TO_G(Y,U,V) (CLAMP (1.164 * (Y - 16) - 0.813 * (V - 128) - 0.391 * (U - 128), 0, 255)) +#define YUV_TO_B(Y,U,V) (CLAMP (1.164 * (Y - 16) + 2.018 * (U - 128), 0, 255)) + +#define A32_COLOR(name, RGB, A, C1, C2, C3) \ +static void \ +fill_color_##name (GstVideoFrame * frame, gint Y, gint U, gint V) \ +{ \ + gint c1, c2, c3; \ + guint32 val; \ + gint width, height; \ + guint8 *dest; \ + \ + dest = GST_VIDEO_FRAME_PLANE_DATA (frame, 0); \ + width = GST_VIDEO_FRAME_COMP_WIDTH (frame, 0); \ + height = GST_VIDEO_FRAME_COMP_HEIGHT (frame, 0); \ + \ + if (RGB) { \ + c1 = YUV_TO_R (Y, U, V); \ + c2 = YUV_TO_G (Y, U, V); \ + c3 = YUV_TO_B (Y, U, V); \ + } else { \ + c1 = Y; \ + c2 = U; \ + c3 = V; \ + } \ + val = GUINT32_FROM_BE ((0xff << A) | (c1 << C1) | (c2 << C2) | (c3 << C3)); \ + \ + compositor_orc_splat_u32 ((guint32 *) dest, val, height * width); \ +} + +A32_COLOR (argb, TRUE, 24, 16, 8, 0); +A32_COLOR (bgra, TRUE, 0, 8, 16, 24); +A32_COLOR (abgr, TRUE, 24, 0, 8, 16); +A32_COLOR (rgba, TRUE, 0, 24, 16, 8); +A32_COLOR (ayuv, FALSE, 24, 16, 8, 0); + +/* Y444, Y42B, I420, YV12, Y41B */ +#define PLANAR_YUV_BLEND(format_name,format_enum,x_round,y_round,MEMCPY,BLENDLOOP) \ +inline static void \ +_blend_##format_name (const guint8 * src, guint8 * dest, \ + gint src_stride, gint dest_stride, gint src_width, gint src_height, \ + gdouble src_alpha) \ +{ \ + gint i; \ + gint b_alpha; \ + \ + /* If it's completely transparent... we just return */ \ + if (G_UNLIKELY (src_alpha == 0.0)) { \ + GST_INFO ("Fast copy (alpha == 0.0)"); \ + return; \ + } \ + \ + /* If it's completely opaque, we do a fast copy */ \ + if (G_UNLIKELY (src_alpha == 1.0)) { \ + GST_INFO ("Fast copy (alpha == 1.0)"); \ + for (i = 0; i < src_height; i++) { \ + MEMCPY (dest, src, src_width); \ + src += src_stride; \ + dest += dest_stride; \ + } \ + return; \ + } \ + \ + b_alpha = CLAMP ((gint) (src_alpha * 256), 0, 256); \ + \ + BLENDLOOP(dest, dest_stride, src, src_stride, b_alpha, src_width, src_height); \ +} \ +\ +static void \ +blend_##format_name (GstVideoFrame * srcframe, gint xpos, gint ypos, \ + gdouble src_alpha, GstVideoFrame * destframe) \ +{ \ + const guint8 *b_src; \ + guint8 *b_dest; \ + gint b_src_width; \ + gint b_src_height; \ + gint xoffset = 0; \ + gint yoffset = 0; \ + gint src_comp_rowstride, dest_comp_rowstride; \ + gint src_comp_height; \ + gint src_comp_width; \ + gint comp_ypos, comp_xpos; \ + gint comp_yoffset, comp_xoffset; \ + gint dest_width, dest_height; \ + const GstVideoFormatInfo *info; \ + gint src_width, src_height; \ + \ + src_width = GST_VIDEO_FRAME_WIDTH (srcframe); \ + src_height = GST_VIDEO_FRAME_HEIGHT (srcframe); \ + \ + info = srcframe->info.finfo; \ + dest_width = GST_VIDEO_FRAME_WIDTH (destframe); \ + dest_height = GST_VIDEO_FRAME_HEIGHT (destframe); \ + \ + xpos = x_round (xpos); \ + ypos = y_round (ypos); \ + \ + b_src_width = src_width; \ + b_src_height = src_height; \ + \ + /* adjust src pointers for negative sizes */ \ + if (xpos < 0) { \ + xoffset = -xpos; \ + b_src_width -= -xpos; \ + xpos = 0; \ + } \ + if (ypos < 0) { \ + yoffset += -ypos; \ + b_src_height -= -ypos; \ + ypos = 0; \ + } \ + /* If x or y offset are larger then the source it's outside of the picture */ \ + if (xoffset > src_width || yoffset > src_height) { \ + return; \ + } \ + \ + /* adjust width/height if the src is bigger than dest */ \ + if (xpos + src_width > dest_width) { \ + b_src_width = dest_width - xpos; \ + } \ + if (ypos + src_height > dest_height) { \ + b_src_height = dest_height - ypos; \ + } \ + if (b_src_width < 0 || b_src_height < 0) { \ + return; \ + } \ + \ + /* First mix Y, then U, then V */ \ + b_src = GST_VIDEO_FRAME_COMP_DATA (srcframe, 0); \ + b_dest = GST_VIDEO_FRAME_COMP_DATA (destframe, 0); \ + src_comp_rowstride = GST_VIDEO_FRAME_COMP_STRIDE (srcframe, 0); \ + dest_comp_rowstride = GST_VIDEO_FRAME_COMP_STRIDE (destframe, 0); \ + src_comp_width = GST_VIDEO_FORMAT_INFO_SCALE_WIDTH(info, 0, b_src_width); \ + src_comp_height = GST_VIDEO_FORMAT_INFO_SCALE_HEIGHT(info, 0, b_src_height); \ + comp_xpos = (xpos == 0) ? 0 : GST_VIDEO_FORMAT_INFO_SCALE_WIDTH (info, 0, xpos); \ + comp_ypos = (ypos == 0) ? 0 : GST_VIDEO_FORMAT_INFO_SCALE_HEIGHT (info, 0, ypos); \ + comp_xoffset = (xoffset == 0) ? 0 : GST_VIDEO_FORMAT_INFO_SCALE_WIDTH (info, 0, xoffset); \ + comp_yoffset = (yoffset == 0) ? 0 : GST_VIDEO_FORMAT_INFO_SCALE_HEIGHT (info, 0, yoffset); \ + _blend_##format_name (b_src + comp_xoffset + comp_yoffset * src_comp_rowstride, \ + b_dest + comp_xpos + comp_ypos * dest_comp_rowstride, \ + src_comp_rowstride, \ + dest_comp_rowstride, src_comp_width, src_comp_height, \ + src_alpha); \ + \ + b_src = GST_VIDEO_FRAME_COMP_DATA (srcframe, 1); \ + b_dest = GST_VIDEO_FRAME_COMP_DATA (destframe, 1); \ + src_comp_rowstride = GST_VIDEO_FRAME_COMP_STRIDE (srcframe, 1); \ + dest_comp_rowstride = GST_VIDEO_FRAME_COMP_STRIDE (destframe, 1); \ + src_comp_width = GST_VIDEO_FORMAT_INFO_SCALE_WIDTH(info, 1, b_src_width); \ + src_comp_height = GST_VIDEO_FORMAT_INFO_SCALE_HEIGHT(info, 1, b_src_height); \ + comp_xpos = (xpos == 0) ? 0 : GST_VIDEO_FORMAT_INFO_SCALE_WIDTH (info, 1, xpos); \ + comp_ypos = (ypos == 0) ? 0 : GST_VIDEO_FORMAT_INFO_SCALE_HEIGHT (info, 1, ypos); \ + comp_xoffset = (xoffset == 0) ? 0 : GST_VIDEO_FORMAT_INFO_SCALE_WIDTH (info, 1, xoffset); \ + comp_yoffset = (yoffset == 0) ? 0 : GST_VIDEO_FORMAT_INFO_SCALE_HEIGHT (info, 1, yoffset); \ + _blend_##format_name (b_src + comp_xoffset + comp_yoffset * src_comp_rowstride, \ + b_dest + comp_xpos + comp_ypos * dest_comp_rowstride, \ + src_comp_rowstride, \ + dest_comp_rowstride, src_comp_width, src_comp_height, \ + src_alpha); \ + \ + b_src = GST_VIDEO_FRAME_COMP_DATA (srcframe, 2); \ + b_dest = GST_VIDEO_FRAME_COMP_DATA (destframe, 2); \ + src_comp_rowstride = GST_VIDEO_FRAME_COMP_STRIDE (srcframe, 2); \ + dest_comp_rowstride = GST_VIDEO_FRAME_COMP_STRIDE (destframe, 2); \ + src_comp_width = GST_VIDEO_FORMAT_INFO_SCALE_WIDTH(info, 2, b_src_width); \ + src_comp_height = GST_VIDEO_FORMAT_INFO_SCALE_HEIGHT(info, 2, b_src_height); \ + comp_xpos = (xpos == 0) ? 0 : GST_VIDEO_FORMAT_INFO_SCALE_WIDTH (info, 2, xpos); \ + comp_ypos = (ypos == 0) ? 0 : GST_VIDEO_FORMAT_INFO_SCALE_HEIGHT (info, 2, ypos); \ + comp_xoffset = (xoffset == 0) ? 0 : GST_VIDEO_FORMAT_INFO_SCALE_WIDTH (info, 2, xoffset); \ + comp_yoffset = (yoffset == 0) ? 0 : GST_VIDEO_FORMAT_INFO_SCALE_HEIGHT (info, 2, yoffset); \ + _blend_##format_name (b_src + comp_xoffset + comp_yoffset * src_comp_rowstride, \ + b_dest + comp_xpos + comp_ypos * dest_comp_rowstride, \ + src_comp_rowstride, \ + dest_comp_rowstride, src_comp_width, src_comp_height, \ + src_alpha); \ +} + +#define PLANAR_YUV_FILL_CHECKER(format_name, format_enum, MEMSET) \ +static void \ +fill_checker_##format_name (GstVideoFrame * frame) \ +{ \ + gint i, j; \ + static const int tab[] = { 80, 160, 80, 160 }; \ + guint8 *p; \ + gint comp_width, comp_height; \ + gint rowstride; \ + \ + p = GST_VIDEO_FRAME_COMP_DATA (frame, 0); \ + comp_width = GST_VIDEO_FRAME_COMP_WIDTH (frame, 0); \ + comp_height = GST_VIDEO_FRAME_COMP_HEIGHT (frame, 0); \ + rowstride = GST_VIDEO_FRAME_COMP_STRIDE (frame, 0); \ + \ + for (i = 0; i < comp_height; i++) { \ + for (j = 0; j < comp_width; j++) { \ + *p++ = tab[((i & 0x8) >> 3) + ((j & 0x8) >> 3)]; \ + } \ + p += rowstride - comp_width; \ + } \ + \ + p = GST_VIDEO_FRAME_COMP_DATA (frame, 1); \ + comp_width = GST_VIDEO_FRAME_COMP_WIDTH (frame, 1); \ + comp_height = GST_VIDEO_FRAME_COMP_HEIGHT (frame, 1); \ + rowstride = GST_VIDEO_FRAME_COMP_STRIDE (frame, 1); \ + \ + for (i = 0; i < comp_height; i++) { \ + MEMSET (p, 0x80, comp_width); \ + p += rowstride; \ + } \ + \ + p = GST_VIDEO_FRAME_COMP_DATA (frame, 2); \ + comp_width = GST_VIDEO_FRAME_COMP_WIDTH (frame, 2); \ + comp_height = GST_VIDEO_FRAME_COMP_HEIGHT (frame, 2); \ + rowstride = GST_VIDEO_FRAME_COMP_STRIDE (frame, 2); \ + \ + for (i = 0; i < comp_height; i++) { \ + MEMSET (p, 0x80, comp_width); \ + p += rowstride; \ + } \ +} + +#define PLANAR_YUV_FILL_COLOR(format_name,format_enum,MEMSET) \ +static void \ +fill_color_##format_name (GstVideoFrame * frame, \ + gint colY, gint colU, gint colV) \ +{ \ + guint8 *p; \ + gint comp_width, comp_height; \ + gint rowstride; \ + gint i; \ + \ + p = GST_VIDEO_FRAME_COMP_DATA (frame, 0); \ + comp_width = GST_VIDEO_FRAME_COMP_WIDTH (frame, 0); \ + comp_height = GST_VIDEO_FRAME_COMP_HEIGHT (frame, 0); \ + rowstride = GST_VIDEO_FRAME_COMP_STRIDE (frame, 0); \ + \ + for (i = 0; i < comp_height; i++) { \ + MEMSET (p, colY, comp_width); \ + p += rowstride; \ + } \ + \ + p = GST_VIDEO_FRAME_COMP_DATA (frame, 1); \ + comp_width = GST_VIDEO_FRAME_COMP_WIDTH (frame, 1); \ + comp_height = GST_VIDEO_FRAME_COMP_HEIGHT (frame, 1); \ + rowstride = GST_VIDEO_FRAME_COMP_STRIDE (frame, 1); \ + \ + for (i = 0; i < comp_height; i++) { \ + MEMSET (p, colU, comp_width); \ + p += rowstride; \ + } \ + \ + p = GST_VIDEO_FRAME_COMP_DATA (frame, 2); \ + comp_width = GST_VIDEO_FRAME_COMP_WIDTH (frame, 2); \ + comp_height = GST_VIDEO_FRAME_COMP_HEIGHT (frame, 2); \ + rowstride = GST_VIDEO_FRAME_COMP_STRIDE (frame, 2); \ + \ + for (i = 0; i < comp_height; i++) { \ + MEMSET (p, colV, comp_width); \ + p += rowstride; \ + } \ +} + +#define GST_ROUND_UP_1(x) (x) + +PLANAR_YUV_BLEND (i420, GST_VIDEO_FORMAT_I420, GST_ROUND_UP_2, + GST_ROUND_UP_2, memcpy, compositor_orc_blend_u8); +PLANAR_YUV_FILL_CHECKER (i420, GST_VIDEO_FORMAT_I420, memset); +PLANAR_YUV_FILL_COLOR (i420, GST_VIDEO_FORMAT_I420, memset); +PLANAR_YUV_FILL_COLOR (yv12, GST_VIDEO_FORMAT_YV12, memset); +PLANAR_YUV_BLEND (y444, GST_VIDEO_FORMAT_Y444, GST_ROUND_UP_1, + GST_ROUND_UP_1, memcpy, compositor_orc_blend_u8); +PLANAR_YUV_FILL_CHECKER (y444, GST_VIDEO_FORMAT_Y444, memset); +PLANAR_YUV_FILL_COLOR (y444, GST_VIDEO_FORMAT_Y444, memset); +PLANAR_YUV_BLEND (y42b, GST_VIDEO_FORMAT_Y42B, GST_ROUND_UP_2, + GST_ROUND_UP_1, memcpy, compositor_orc_blend_u8); +PLANAR_YUV_FILL_CHECKER (y42b, GST_VIDEO_FORMAT_Y42B, memset); +PLANAR_YUV_FILL_COLOR (y42b, GST_VIDEO_FORMAT_Y42B, memset); +PLANAR_YUV_BLEND (y41b, GST_VIDEO_FORMAT_Y41B, GST_ROUND_UP_4, + GST_ROUND_UP_1, memcpy, compositor_orc_blend_u8); +PLANAR_YUV_FILL_CHECKER (y41b, GST_VIDEO_FORMAT_Y41B, memset); +PLANAR_YUV_FILL_COLOR (y41b, GST_VIDEO_FORMAT_Y41B, memset); + +/* NV12, NV21 */ +#define NV_YUV_BLEND(format_name,MEMCPY,BLENDLOOP) \ +inline static void \ +_blend_##format_name (const guint8 * src, guint8 * dest, \ + gint src_stride, gint dest_stride, gint src_width, gint src_height, \ + gdouble src_alpha) \ +{ \ + gint i; \ + gint b_alpha; \ + \ + /* If it's completely transparent... we just return */ \ + if (G_UNLIKELY (src_alpha == 0.0)) { \ + GST_INFO ("Fast copy (alpha == 0.0)"); \ + return; \ + } \ + \ + /* If it's completely opaque, we do a fast copy */ \ + if (G_UNLIKELY (src_alpha == 1.0)) { \ + GST_INFO ("Fast copy (alpha == 1.0)"); \ + for (i = 0; i < src_height; i++) { \ + MEMCPY (dest, src, src_width); \ + src += src_stride; \ + dest += dest_stride; \ + } \ + return; \ + } \ + \ + b_alpha = CLAMP ((gint) (src_alpha * 256), 0, 256); \ + \ + BLENDLOOP(dest, dest_stride, src, src_stride, b_alpha, src_width, src_height); \ +} \ +\ +static void \ +blend_##format_name (GstVideoFrame * srcframe, gint xpos, gint ypos, \ + gdouble src_alpha, GstVideoFrame * destframe) \ +{ \ + const guint8 *b_src; \ + guint8 *b_dest; \ + gint b_src_width; \ + gint b_src_height; \ + gint xoffset = 0; \ + gint yoffset = 0; \ + gint src_comp_rowstride, dest_comp_rowstride; \ + gint src_comp_height; \ + gint src_comp_width; \ + gint comp_ypos, comp_xpos; \ + gint comp_yoffset, comp_xoffset; \ + gint dest_width, dest_height; \ + const GstVideoFormatInfo *info; \ + gint src_width, src_height; \ + \ + src_width = GST_VIDEO_FRAME_WIDTH (srcframe); \ + src_height = GST_VIDEO_FRAME_HEIGHT (srcframe); \ + \ + info = srcframe->info.finfo; \ + dest_width = GST_VIDEO_FRAME_WIDTH (destframe); \ + dest_height = GST_VIDEO_FRAME_HEIGHT (destframe); \ + \ + xpos = GST_ROUND_UP_2 (xpos); \ + ypos = GST_ROUND_UP_2 (ypos); \ + \ + b_src_width = src_width; \ + b_src_height = src_height; \ + \ + /* adjust src pointers for negative sizes */ \ + if (xpos < 0) { \ + xoffset = -xpos; \ + b_src_width -= -xpos; \ + xpos = 0; \ + } \ + if (ypos < 0) { \ + yoffset += -ypos; \ + b_src_height -= -ypos; \ + ypos = 0; \ + } \ + /* If x or y offset are larger then the source it's outside of the picture */ \ + if (xoffset > src_width || yoffset > src_height) { \ + return; \ + } \ + \ + /* adjust width/height if the src is bigger than dest */ \ + if (xpos + src_width > dest_width) { \ + b_src_width = dest_width - xpos; \ + } \ + if (ypos + src_height > dest_height) { \ + b_src_height = dest_height - ypos; \ + } \ + if (b_src_width < 0 || b_src_height < 0) { \ + return; \ + } \ + \ + /* First mix Y, then UV */ \ + b_src = GST_VIDEO_FRAME_COMP_DATA (srcframe, 0); \ + b_dest = GST_VIDEO_FRAME_COMP_DATA (destframe, 0); \ + src_comp_rowstride = GST_VIDEO_FRAME_COMP_STRIDE (srcframe, 0); \ + dest_comp_rowstride = GST_VIDEO_FRAME_COMP_STRIDE (destframe, 0); \ + src_comp_width = GST_VIDEO_FORMAT_INFO_SCALE_WIDTH(info, 0, b_src_width); \ + src_comp_height = GST_VIDEO_FORMAT_INFO_SCALE_HEIGHT(info, 0, b_src_height); \ + comp_xpos = (xpos == 0) ? 0 : GST_VIDEO_FORMAT_INFO_SCALE_WIDTH (info, 0, xpos); \ + comp_ypos = (ypos == 0) ? 0 : GST_VIDEO_FORMAT_INFO_SCALE_HEIGHT (info, 0, ypos); \ + comp_xoffset = (xoffset == 0) ? 0 : GST_VIDEO_FORMAT_INFO_SCALE_WIDTH (info, 0, xoffset); \ + comp_yoffset = (yoffset == 0) ? 0 : GST_VIDEO_FORMAT_INFO_SCALE_HEIGHT (info, 0, yoffset); \ + _blend_##format_name (b_src + comp_xoffset + comp_yoffset * src_comp_rowstride, \ + b_dest + comp_xpos + comp_ypos * dest_comp_rowstride, \ + src_comp_rowstride, \ + dest_comp_rowstride, src_comp_width, src_comp_height, \ + src_alpha); \ + \ + b_src = GST_VIDEO_FRAME_PLANE_DATA (srcframe, 1); \ + b_dest = GST_VIDEO_FRAME_PLANE_DATA (destframe, 1); \ + src_comp_rowstride = GST_VIDEO_FRAME_COMP_STRIDE (srcframe, 1); \ + dest_comp_rowstride = GST_VIDEO_FRAME_COMP_STRIDE (destframe, 1); \ + src_comp_width = GST_VIDEO_FORMAT_INFO_SCALE_WIDTH(info, 1, b_src_width); \ + src_comp_height = GST_VIDEO_FORMAT_INFO_SCALE_HEIGHT(info, 1, b_src_height); \ + comp_xpos = (xpos == 0) ? 0 : GST_VIDEO_FORMAT_INFO_SCALE_WIDTH (info, 1, xpos); \ + comp_ypos = (ypos == 0) ? 0 : GST_VIDEO_FORMAT_INFO_SCALE_HEIGHT (info, 1, ypos); \ + comp_xoffset = (xoffset == 0) ? 0 : GST_VIDEO_FORMAT_INFO_SCALE_WIDTH (info, 1, xoffset); \ + comp_yoffset = (yoffset == 0) ? 0 : GST_VIDEO_FORMAT_INFO_SCALE_HEIGHT (info, 1, yoffset); \ + _blend_##format_name (b_src + comp_xoffset * 2 + comp_yoffset * src_comp_rowstride, \ + b_dest + comp_xpos * 2 + comp_ypos * dest_comp_rowstride, \ + src_comp_rowstride, \ + dest_comp_rowstride, 2 * src_comp_width, src_comp_height, \ + src_alpha); \ +} + +#define NV_YUV_FILL_CHECKER(format_name, MEMSET) \ +static void \ +fill_checker_##format_name (GstVideoFrame * frame) \ +{ \ + gint i, j; \ + static const int tab[] = { 80, 160, 80, 160 }; \ + guint8 *p; \ + gint comp_width, comp_height; \ + gint rowstride; \ + \ + p = GST_VIDEO_FRAME_COMP_DATA (frame, 0); \ + comp_width = GST_VIDEO_FRAME_COMP_WIDTH (frame, 0); \ + comp_height = GST_VIDEO_FRAME_COMP_HEIGHT (frame, 0); \ + rowstride = GST_VIDEO_FRAME_COMP_STRIDE (frame, 0); \ + \ + for (i = 0; i < comp_height; i++) { \ + for (j = 0; j < comp_width; j++) { \ + *p++ = tab[((i & 0x8) >> 3) + ((j & 0x8) >> 3)]; \ + } \ + p += rowstride - comp_width; \ + } \ + \ + p = GST_VIDEO_FRAME_PLANE_DATA (frame, 1); \ + comp_width = GST_VIDEO_FRAME_COMP_WIDTH (frame, 1); \ + comp_height = GST_VIDEO_FRAME_COMP_HEIGHT (frame, 1); \ + rowstride = GST_VIDEO_FRAME_COMP_STRIDE (frame, 1); \ + \ + for (i = 0; i < comp_height; i++) { \ + MEMSET (p, 0x80, comp_width * 2); \ + p += rowstride; \ + } \ +} + +#define NV_YUV_FILL_COLOR(format_name,MEMSET) \ +static void \ +fill_color_##format_name (GstVideoFrame * frame, \ + gint colY, gint colU, gint colV) \ +{ \ + guint8 *y, *u, *v; \ + gint comp_width, comp_height; \ + gint rowstride; \ + gint i, j; \ + \ + y = GST_VIDEO_FRAME_COMP_DATA (frame, 0); \ + comp_width = GST_VIDEO_FRAME_COMP_WIDTH (frame, 0); \ + comp_height = GST_VIDEO_FRAME_COMP_HEIGHT (frame, 0); \ + rowstride = GST_VIDEO_FRAME_COMP_STRIDE (frame, 0); \ + \ + for (i = 0; i < comp_height; i++) { \ + MEMSET (y, colY, comp_width); \ + y += rowstride; \ + } \ + \ + u = GST_VIDEO_FRAME_COMP_DATA (frame, 1); \ + v = GST_VIDEO_FRAME_COMP_DATA (frame, 2); \ + comp_width = GST_VIDEO_FRAME_COMP_WIDTH (frame, 1); \ + comp_height = GST_VIDEO_FRAME_COMP_HEIGHT (frame, 1); \ + rowstride = GST_VIDEO_FRAME_COMP_STRIDE (frame, 1); \ + \ + for (i = 0; i < comp_height; i++) { \ + for (j = 0; j < comp_width; j++) { \ + u[j*2] = colU; \ + v[j*2] = colV; \ + } \ + u += rowstride; \ + v += rowstride; \ + } \ +} + +NV_YUV_BLEND (nv12, memcpy, compositor_orc_blend_u8); +NV_YUV_FILL_CHECKER (nv12, memset); +NV_YUV_FILL_COLOR (nv12, memset); +NV_YUV_BLEND (nv21, memcpy, compositor_orc_blend_u8); +NV_YUV_FILL_CHECKER (nv21, memset); + +/* RGB, BGR, xRGB, xBGR, RGBx, BGRx */ + +#define RGB_BLEND(name, bpp, MEMCPY, BLENDLOOP) \ +static void \ +blend_##name (GstVideoFrame * srcframe, gint xpos, gint ypos, \ + gdouble src_alpha, GstVideoFrame * destframe) \ +{ \ + gint b_alpha; \ + gint i; \ + gint src_stride, dest_stride; \ + gint dest_width, dest_height; \ + guint8 *dest, *src; \ + gint src_width, src_height; \ + \ + src_width = GST_VIDEO_FRAME_WIDTH (srcframe); \ + src_height = GST_VIDEO_FRAME_HEIGHT (srcframe); \ + \ + src = GST_VIDEO_FRAME_PLANE_DATA (srcframe, 0); \ + dest = GST_VIDEO_FRAME_PLANE_DATA (destframe, 0); \ + \ + dest_width = GST_VIDEO_FRAME_WIDTH (destframe); \ + dest_height = GST_VIDEO_FRAME_HEIGHT (destframe); \ + \ + src_stride = GST_VIDEO_FRAME_COMP_STRIDE (srcframe, 0); \ + dest_stride = GST_VIDEO_FRAME_COMP_STRIDE (destframe, 0); \ + \ + b_alpha = CLAMP ((gint) (src_alpha * 256), 0, 256); \ + \ + /* adjust src pointers for negative sizes */ \ + if (xpos < 0) { \ + src += -xpos * bpp; \ + src_width -= -xpos; \ + xpos = 0; \ + } \ + if (ypos < 0) { \ + src += -ypos * src_stride; \ + src_height -= -ypos; \ + ypos = 0; \ + } \ + /* adjust width/height if the src is bigger than dest */ \ + if (xpos + src_width > dest_width) { \ + src_width = dest_width - xpos; \ + } \ + if (ypos + src_height > dest_height) { \ + src_height = dest_height - ypos; \ + } \ + \ + dest = dest + bpp * xpos + (ypos * dest_stride); \ + /* If it's completely transparent... we just return */ \ + if (G_UNLIKELY (src_alpha == 0.0)) { \ + GST_INFO ("Fast copy (alpha == 0.0)"); \ + return; \ + } \ + \ + /* If it's completely opaque, we do a fast copy */ \ + if (G_UNLIKELY (src_alpha == 1.0)) { \ + GST_INFO ("Fast copy (alpha == 1.0)"); \ + for (i = 0; i < src_height; i++) { \ + MEMCPY (dest, src, bpp * src_width); \ + src += src_stride; \ + dest += dest_stride; \ + } \ + return; \ + } \ + \ + BLENDLOOP(dest, dest_stride, src, src_stride, b_alpha, src_width * bpp, src_height); \ +} + +#define RGB_FILL_CHECKER_C(name, bpp, r, g, b) \ +static void \ +fill_checker_##name##_c (GstVideoFrame * frame) \ +{ \ + gint i, j; \ + static const int tab[] = { 80, 160, 80, 160 }; \ + gint stride, dest_add, width, height; \ + guint8 *dest; \ + \ + width = GST_VIDEO_FRAME_WIDTH (frame); \ + height = GST_VIDEO_FRAME_HEIGHT (frame); \ + dest = GST_VIDEO_FRAME_PLANE_DATA (frame, 0); \ + stride = GST_VIDEO_FRAME_COMP_STRIDE (frame, 0); \ + dest_add = stride - width * bpp; \ + \ + for (i = 0; i < height; i++) { \ + for (j = 0; j < width; j++) { \ + dest[r] = tab[((i & 0x8) >> 3) + ((j & 0x8) >> 3)]; /* red */ \ + dest[g] = tab[((i & 0x8) >> 3) + ((j & 0x8) >> 3)]; /* green */ \ + dest[b] = tab[((i & 0x8) >> 3) + ((j & 0x8) >> 3)]; /* blue */ \ + dest += bpp; \ + } \ + dest += dest_add; \ + } \ +} + +#define RGB_FILL_COLOR(name, bpp, MEMSET_RGB) \ +static void \ +fill_color_##name (GstVideoFrame * frame, \ + gint colY, gint colU, gint colV) \ +{ \ + gint red, green, blue; \ + gint i; \ + gint dest_stride; \ + gint width, height; \ + guint8 *dest; \ + \ + width = GST_VIDEO_FRAME_WIDTH (frame); \ + height = GST_VIDEO_FRAME_HEIGHT (frame); \ + dest = GST_VIDEO_FRAME_PLANE_DATA (frame, 0); \ + dest_stride = GST_VIDEO_FRAME_COMP_STRIDE (frame, 0); \ + \ + red = YUV_TO_R (colY, colU, colV); \ + green = YUV_TO_G (colY, colU, colV); \ + blue = YUV_TO_B (colY, colU, colV); \ + \ + for (i = 0; i < height; i++) { \ + MEMSET_RGB (dest, red, green, blue, width); \ + dest += dest_stride; \ + } \ +} + +#define MEMSET_RGB_C(name, r, g, b) \ +static inline void \ +_memset_##name##_c (guint8* dest, gint red, gint green, gint blue, gint width) { \ + gint j; \ + \ + for (j = 0; j < width; j++) { \ + dest[r] = red; \ + dest[g] = green; \ + dest[b] = blue; \ + dest += 3; \ + } \ +} + +#define MEMSET_XRGB(name, r, g, b) \ +static inline void \ +_memset_##name (guint8* dest, gint red, gint green, gint blue, gint width) { \ + guint32 val; \ + \ + val = GUINT32_FROM_BE ((red << r) | (green << g) | (blue << b)); \ + compositor_orc_splat_u32 ((guint32 *) dest, val, width); \ +} + +#define _orc_memcpy_u32(dest,src,len) compositor_orc_memcpy_u32((guint32 *) dest, (const guint32 *) src, len/4) + +RGB_BLEND (rgb, 3, memcpy, compositor_orc_blend_u8); +RGB_FILL_CHECKER_C (rgb, 3, 0, 1, 2); +MEMSET_RGB_C (rgb, 0, 1, 2); +RGB_FILL_COLOR (rgb_c, 3, _memset_rgb_c); + +MEMSET_RGB_C (bgr, 2, 1, 0); +RGB_FILL_COLOR (bgr_c, 3, _memset_bgr_c); + +RGB_BLEND (xrgb, 4, _orc_memcpy_u32, compositor_orc_blend_u8); +RGB_FILL_CHECKER_C (xrgb, 4, 1, 2, 3); +MEMSET_XRGB (xrgb, 24, 16, 0); +RGB_FILL_COLOR (xrgb, 4, _memset_xrgb); + +MEMSET_XRGB (xbgr, 0, 16, 24); +RGB_FILL_COLOR (xbgr, 4, _memset_xbgr); + +MEMSET_XRGB (rgbx, 24, 16, 8); +RGB_FILL_COLOR (rgbx, 4, _memset_rgbx); + +MEMSET_XRGB (bgrx, 8, 16, 24); +RGB_FILL_COLOR (bgrx, 4, _memset_bgrx); + +/* YUY2, YVYU, UYVY */ + +#define PACKED_422_BLEND(name, MEMCPY, BLENDLOOP) \ +static void \ +blend_##name (GstVideoFrame * srcframe, gint xpos, gint ypos, \ + gdouble src_alpha, GstVideoFrame * destframe) \ +{ \ + gint b_alpha; \ + gint i; \ + gint src_stride, dest_stride; \ + gint dest_width, dest_height; \ + guint8 *src, *dest; \ + gint src_width, src_height; \ + \ + src_width = GST_VIDEO_FRAME_WIDTH (srcframe); \ + src_height = GST_VIDEO_FRAME_HEIGHT (srcframe); \ + \ + dest_width = GST_VIDEO_FRAME_WIDTH (destframe); \ + dest_height = GST_VIDEO_FRAME_HEIGHT (destframe); \ + \ + src = GST_VIDEO_FRAME_PLANE_DATA (srcframe, 0); \ + dest = GST_VIDEO_FRAME_PLANE_DATA (destframe, 0); \ + \ + src_stride = GST_VIDEO_FRAME_COMP_STRIDE (srcframe, 0); \ + dest_stride = GST_VIDEO_FRAME_COMP_STRIDE (destframe, 0); \ + \ + b_alpha = CLAMP ((gint) (src_alpha * 256), 0, 256); \ + \ + xpos = GST_ROUND_UP_2 (xpos); \ + \ + /* adjust src pointers for negative sizes */ \ + if (xpos < 0) { \ + src += -xpos * 2; \ + src_width -= -xpos; \ + xpos = 0; \ + } \ + if (ypos < 0) { \ + src += -ypos * src_stride; \ + src_height -= -ypos; \ + ypos = 0; \ + } \ + \ + /* adjust width/height if the src is bigger than dest */ \ + if (xpos + src_width > dest_width) { \ + src_width = dest_width - xpos; \ + } \ + if (ypos + src_height > dest_height) { \ + src_height = dest_height - ypos; \ + } \ + \ + dest = dest + 2 * xpos + (ypos * dest_stride); \ + /* If it's completely transparent... we just return */ \ + if (G_UNLIKELY (src_alpha == 0.0)) { \ + GST_INFO ("Fast copy (alpha == 0.0)"); \ + return; \ + } \ + \ + /* If it's completely opaque, we do a fast copy */ \ + if (G_UNLIKELY (src_alpha == 1.0)) { \ + GST_INFO ("Fast copy (alpha == 1.0)"); \ + for (i = 0; i < src_height; i++) { \ + MEMCPY (dest, src, 2 * src_width); \ + src += src_stride; \ + dest += dest_stride; \ + } \ + return; \ + } \ + \ + BLENDLOOP(dest, dest_stride, src, src_stride, b_alpha, 2 * src_width, src_height); \ +} + +#define PACKED_422_FILL_CHECKER_C(name, Y1, U, Y2, V) \ +static void \ +fill_checker_##name##_c (GstVideoFrame * frame) \ +{ \ + gint i, j; \ + static const int tab[] = { 80, 160, 80, 160 }; \ + gint dest_add; \ + gint width, height; \ + guint8 *dest; \ + \ + width = GST_VIDEO_FRAME_WIDTH (frame); \ + width = GST_ROUND_UP_2 (width); \ + height = GST_VIDEO_FRAME_HEIGHT (frame); \ + dest = GST_VIDEO_FRAME_PLANE_DATA (frame, 0); \ + dest_add = GST_VIDEO_FRAME_COMP_STRIDE (frame, 0) - width * 2; \ + width /= 2; \ + \ + for (i = 0; i < height; i++) { \ + for (j = 0; j < width; j++) { \ + dest[Y1] = tab[((i & 0x8) >> 3) + ((j & 0x8) >> 3)]; \ + dest[Y2] = tab[((i & 0x8) >> 3) + ((j & 0x8) >> 3)]; \ + dest[U] = 128; \ + dest[V] = 128; \ + dest += 4; \ + } \ + dest += dest_add; \ + } \ +} + +#define PACKED_422_FILL_COLOR(name, Y1, U, Y2, V) \ +static void \ +fill_color_##name (GstVideoFrame * frame, \ + gint colY, gint colU, gint colV) \ +{ \ + gint i; \ + gint dest_stride; \ + guint32 val; \ + gint width, height; \ + guint8 *dest; \ + \ + width = GST_VIDEO_FRAME_WIDTH (frame); \ + width = GST_ROUND_UP_2 (width); \ + height = GST_VIDEO_FRAME_HEIGHT (frame); \ + dest = GST_VIDEO_FRAME_PLANE_DATA (frame, 0); \ + dest_stride = GST_VIDEO_FRAME_COMP_STRIDE (frame, 0); \ + width /= 2; \ + \ + val = GUINT32_FROM_BE ((colY << Y1) | (colY << Y2) | (colU << U) | (colV << V)); \ + \ + for (i = 0; i < height; i++) { \ + compositor_orc_splat_u32 ((guint32 *) dest, val, width); \ + dest += dest_stride; \ + } \ +} + +PACKED_422_BLEND (yuy2, memcpy, compositor_orc_blend_u8); +PACKED_422_FILL_CHECKER_C (yuy2, 0, 1, 2, 3); +PACKED_422_FILL_CHECKER_C (uyvy, 1, 0, 3, 2); +PACKED_422_FILL_COLOR (yuy2, 24, 16, 8, 0); +PACKED_422_FILL_COLOR (yvyu, 24, 0, 8, 16); +PACKED_422_FILL_COLOR (uyvy, 16, 24, 0, 8); + +/* Init function */ +BlendFunction gst_compositor_blend_argb; +BlendFunction gst_compositor_blend_bgra; +BlendFunction gst_compositor_overlay_argb; +BlendFunction gst_compositor_overlay_bgra; +/* AYUV/ABGR is equal to ARGB, RGBA is equal to BGRA */ +BlendFunction gst_compositor_blend_y444; +BlendFunction gst_compositor_blend_y42b; +BlendFunction gst_compositor_blend_i420; +/* I420 is equal to YV12 */ +BlendFunction gst_compositor_blend_nv12; +BlendFunction gst_compositor_blend_nv21; +BlendFunction gst_compositor_blend_y41b; +BlendFunction gst_compositor_blend_rgb; +/* BGR is equal to RGB */ +BlendFunction gst_compositor_blend_rgbx; +/* BGRx, xRGB, xBGR are equal to RGBx */ +BlendFunction gst_compositor_blend_yuy2; +/* YVYU and UYVY are equal to YUY2 */ + +FillCheckerFunction gst_compositor_fill_checker_argb; +FillCheckerFunction gst_compositor_fill_checker_bgra; +/* ABGR is equal to ARGB, RGBA is equal to BGRA */ +FillCheckerFunction gst_compositor_fill_checker_ayuv; +FillCheckerFunction gst_compositor_fill_checker_y444; +FillCheckerFunction gst_compositor_fill_checker_y42b; +FillCheckerFunction gst_compositor_fill_checker_i420; +/* I420 is equal to YV12 */ +FillCheckerFunction gst_compositor_fill_checker_nv12; +FillCheckerFunction gst_compositor_fill_checker_nv21; +FillCheckerFunction gst_compositor_fill_checker_y41b; +FillCheckerFunction gst_compositor_fill_checker_rgb; +/* BGR is equal to RGB */ +FillCheckerFunction gst_compositor_fill_checker_xrgb; +/* BGRx, xRGB, xBGR are equal to RGBx */ +FillCheckerFunction gst_compositor_fill_checker_yuy2; +/* YVYU is equal to YUY2 */ +FillCheckerFunction gst_compositor_fill_checker_uyvy; + +FillColorFunction gst_compositor_fill_color_argb; +FillColorFunction gst_compositor_fill_color_bgra; +FillColorFunction gst_compositor_fill_color_abgr; +FillColorFunction gst_compositor_fill_color_rgba; +FillColorFunction gst_compositor_fill_color_ayuv; +FillColorFunction gst_compositor_fill_color_y444; +FillColorFunction gst_compositor_fill_color_y42b; +FillColorFunction gst_compositor_fill_color_i420; +FillColorFunction gst_compositor_fill_color_yv12; +FillColorFunction gst_compositor_fill_color_nv12; +/* NV21 is equal to NV12 */ +FillColorFunction gst_compositor_fill_color_y41b; +FillColorFunction gst_compositor_fill_color_rgb; +FillColorFunction gst_compositor_fill_color_bgr; +FillColorFunction gst_compositor_fill_color_xrgb; +FillColorFunction gst_compositor_fill_color_xbgr; +FillColorFunction gst_compositor_fill_color_rgbx; +FillColorFunction gst_compositor_fill_color_bgrx; +FillColorFunction gst_compositor_fill_color_yuy2; +FillColorFunction gst_compositor_fill_color_yvyu; +FillColorFunction gst_compositor_fill_color_uyvy; + +void +gst_compositor_init_blend (void) +{ + GST_DEBUG_CATEGORY_INIT (gst_compositor_blend_debug, "compositor_blend", 0, + "video compositor blending functions"); + + gst_compositor_blend_argb = blend_argb; + gst_compositor_blend_bgra = blend_bgra; + gst_compositor_overlay_argb = overlay_argb; + gst_compositor_overlay_bgra = overlay_bgra; + gst_compositor_blend_i420 = blend_i420; + gst_compositor_blend_nv12 = blend_nv12; + gst_compositor_blend_nv21 = blend_nv21; + gst_compositor_blend_y444 = blend_y444; + gst_compositor_blend_y42b = blend_y42b; + gst_compositor_blend_y41b = blend_y41b; + gst_compositor_blend_rgb = blend_rgb; + gst_compositor_blend_xrgb = blend_xrgb; + gst_compositor_blend_yuy2 = blend_yuy2; + + gst_compositor_fill_checker_argb = fill_checker_argb_c; + gst_compositor_fill_checker_bgra = fill_checker_bgra_c; + gst_compositor_fill_checker_ayuv = fill_checker_ayuv_c; + gst_compositor_fill_checker_i420 = fill_checker_i420; + gst_compositor_fill_checker_nv12 = fill_checker_nv12; + gst_compositor_fill_checker_nv21 = fill_checker_nv21; + gst_compositor_fill_checker_y444 = fill_checker_y444; + gst_compositor_fill_checker_y42b = fill_checker_y42b; + gst_compositor_fill_checker_y41b = fill_checker_y41b; + gst_compositor_fill_checker_rgb = fill_checker_rgb_c; + gst_compositor_fill_checker_xrgb = fill_checker_xrgb_c; + gst_compositor_fill_checker_yuy2 = fill_checker_yuy2_c; + gst_compositor_fill_checker_uyvy = fill_checker_uyvy_c; + + gst_compositor_fill_color_argb = fill_color_argb; + gst_compositor_fill_color_bgra = fill_color_bgra; + gst_compositor_fill_color_abgr = fill_color_abgr; + gst_compositor_fill_color_rgba = fill_color_rgba; + gst_compositor_fill_color_ayuv = fill_color_ayuv; + gst_compositor_fill_color_i420 = fill_color_i420; + gst_compositor_fill_color_yv12 = fill_color_yv12; + gst_compositor_fill_color_nv12 = fill_color_nv12; + gst_compositor_fill_color_y444 = fill_color_y444; + gst_compositor_fill_color_y42b = fill_color_y42b; + gst_compositor_fill_color_y41b = fill_color_y41b; + gst_compositor_fill_color_rgb = fill_color_rgb_c; + gst_compositor_fill_color_bgr = fill_color_bgr_c; + gst_compositor_fill_color_xrgb = fill_color_xrgb; + gst_compositor_fill_color_xbgr = fill_color_xbgr; + gst_compositor_fill_color_rgbx = fill_color_rgbx; + gst_compositor_fill_color_bgrx = fill_color_bgrx; + gst_compositor_fill_color_yuy2 = fill_color_yuy2; + gst_compositor_fill_color_yvyu = fill_color_yvyu; + gst_compositor_fill_color_uyvy = fill_color_uyvy; +} diff --git a/gst/compositor/blend.h b/gst/compositor/blend.h new file mode 100644 index 0000000000..1cc127c4cf --- /dev/null +++ b/gst/compositor/blend.h @@ -0,0 +1,103 @@ +/* + * Copyright (C) 2009 Sebastian Dröge + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifndef __BLEND_H__ +#define __BLEND_H__ + +#include +#include + +typedef void (*BlendFunction) (GstVideoFrame *srcframe, gint xpos, gint ypos, gdouble src_alpha, GstVideoFrame * destframe); +typedef void (*FillCheckerFunction) (GstVideoFrame * frame); +typedef void (*FillColorFunction) (GstVideoFrame * frame, gint c1, gint c2, gint c3); + +extern BlendFunction gst_compositor_blend_argb; +extern BlendFunction gst_compositor_blend_bgra; +#define gst_compositor_blend_ayuv gst_compositor_blend_argb +#define gst_compositor_blend_abgr gst_compositor_blend_argb +#define gst_compositor_blend_rgba gst_compositor_blend_bgra +extern BlendFunction gst_compositor_overlay_argb; +extern BlendFunction gst_compositor_overlay_bgra; +#define gst_compositor_overlay_ayuv gst_compositor_overlay_argb +#define gst_compositor_overlay_abgr gst_compositor_overlay_argb +#define gst_compositor_overlay_rgba gst_compositor_overlay_bgra +extern BlendFunction gst_compositor_blend_i420; +#define gst_compositor_blend_yv12 gst_compositor_blend_i420 +extern BlendFunction gst_compositor_blend_nv12; +extern BlendFunction gst_compositor_blend_nv21; +extern BlendFunction gst_compositor_blend_y41b; +extern BlendFunction gst_compositor_blend_y42b; +extern BlendFunction gst_compositor_blend_y444; +extern BlendFunction gst_compositor_blend_rgb; +#define gst_compositor_blend_bgr gst_compositor_blend_rgb +extern BlendFunction gst_compositor_blend_rgbx; +#define gst_compositor_blend_bgrx gst_compositor_blend_rgbx +#define gst_compositor_blend_xrgb gst_compositor_blend_rgbx +#define gst_compositor_blend_xbgr gst_compositor_blend_rgbx +extern BlendFunction gst_compositor_blend_yuy2; +#define gst_compositor_blend_uyvy gst_compositor_blend_yuy2; +#define gst_compositor_blend_yvyu gst_compositor_blend_yuy2; + +extern FillCheckerFunction gst_compositor_fill_checker_argb; +#define gst_compositor_fill_checker_abgr gst_compositor_fill_checker_argb +extern FillCheckerFunction gst_compositor_fill_checker_bgra; +#define gst_compositor_fill_checker_rgba gst_compositor_fill_checker_bgra +extern FillCheckerFunction gst_compositor_fill_checker_ayuv; +extern FillCheckerFunction gst_compositor_fill_checker_i420; +#define gst_compositor_fill_checker_yv12 gst_compositor_fill_checker_i420 +extern FillCheckerFunction gst_compositor_fill_checker_nv12; +extern FillCheckerFunction gst_compositor_fill_checker_nv21; +extern FillCheckerFunction gst_compositor_fill_checker_y41b; +extern FillCheckerFunction gst_compositor_fill_checker_y42b; +extern FillCheckerFunction gst_compositor_fill_checker_y444; +extern FillCheckerFunction gst_compositor_fill_checker_rgb; +#define gst_compositor_fill_checker_bgr gst_compositor_fill_checker_rgb +extern FillCheckerFunction gst_compositor_fill_checker_rgbx; +#define gst_compositor_fill_checker_bgrx gst_compositor_fill_checker_rgbx +#define gst_compositor_fill_checker_xrgb gst_compositor_fill_checker_rgbx +#define gst_compositor_fill_checker_xbgr gst_compositor_fill_checker_rgbx +extern FillCheckerFunction gst_compositor_fill_checker_yuy2; +#define gst_compositor_fill_checker_yvyu gst_compositor_fill_checker_yuy2; +extern FillCheckerFunction gst_compositor_fill_checker_uyvy; + +extern FillColorFunction gst_compositor_fill_color_argb; +extern FillColorFunction gst_compositor_fill_color_abgr; +extern FillColorFunction gst_compositor_fill_color_bgra; +extern FillColorFunction gst_compositor_fill_color_rgba; +extern FillColorFunction gst_compositor_fill_color_ayuv; +extern FillColorFunction gst_compositor_fill_color_i420; +extern FillColorFunction gst_compositor_fill_color_yv12; +extern FillColorFunction gst_compositor_fill_color_nv12; +#define gst_compositor_fill_color_nv21 gst_compositor_fill_color_nv12; +extern FillColorFunction gst_compositor_fill_color_y41b; +extern FillColorFunction gst_compositor_fill_color_y42b; +extern FillColorFunction gst_compositor_fill_color_y444; +extern FillColorFunction gst_compositor_fill_color_rgb; +extern FillColorFunction gst_compositor_fill_color_bgr; +extern FillColorFunction gst_compositor_fill_color_xrgb; +extern FillColorFunction gst_compositor_fill_color_xbgr; +extern FillColorFunction gst_compositor_fill_color_rgbx; +extern FillColorFunction gst_compositor_fill_color_bgrx; +extern FillColorFunction gst_compositor_fill_color_yuy2; +extern FillColorFunction gst_compositor_fill_color_yvyu; +extern FillColorFunction gst_compositor_fill_color_uyvy; + +void gst_compositor_init_blend (void); + +#endif /* __BLEND_H__ */ diff --git a/gst/compositor/blendorc.h b/gst/compositor/blendorc.h new file mode 100644 index 0000000000..6a39edc7f8 --- /dev/null +++ b/gst/compositor/blendorc.h @@ -0,0 +1,96 @@ + +/* autogenerated from blendorc.orc */ + +#ifndef _BLENDORC_H_ +#define _BLENDORC_H_ + +#include + +#ifdef __cplusplus +extern "C" { +#endif + + + +#ifndef _ORC_INTEGER_TYPEDEFS_ +#define _ORC_INTEGER_TYPEDEFS_ +#if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L +#include +typedef int8_t orc_int8; +typedef int16_t orc_int16; +typedef int32_t orc_int32; +typedef int64_t orc_int64; +typedef uint8_t orc_uint8; +typedef uint16_t orc_uint16; +typedef uint32_t orc_uint32; +typedef uint64_t orc_uint64; +#define ORC_UINT64_C(x) UINT64_C(x) +#elif defined(_MSC_VER) +typedef signed __int8 orc_int8; +typedef signed __int16 orc_int16; +typedef signed __int32 orc_int32; +typedef signed __int64 orc_int64; +typedef unsigned __int8 orc_uint8; +typedef unsigned __int16 orc_uint16; +typedef unsigned __int32 orc_uint32; +typedef unsigned __int64 orc_uint64; +#define ORC_UINT64_C(x) (x##Ui64) +#define inline __inline +#else +#include +typedef signed char orc_int8; +typedef short orc_int16; +typedef int orc_int32; +typedef unsigned char orc_uint8; +typedef unsigned short orc_uint16; +typedef unsigned int orc_uint32; +#if INT_MAX == LONG_MAX +typedef long long orc_int64; +typedef unsigned long long orc_uint64; +#define ORC_UINT64_C(x) (x##ULL) +#else +typedef long orc_int64; +typedef unsigned long orc_uint64; +#define ORC_UINT64_C(x) (x##UL) +#endif +#endif +typedef union { orc_int16 i; orc_int8 x2[2]; } orc_union16; +typedef union { orc_int32 i; float f; orc_int16 x2[2]; orc_int8 x4[4]; } orc_union32; +typedef union { orc_int64 i; double f; orc_int32 x2[2]; float x2f[2]; orc_int16 x4[4]; } orc_union64; +#endif +#ifndef ORC_RESTRICT +#if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L +#define ORC_RESTRICT restrict +#elif defined(__GNUC__) && __GNUC__ >= 4 +#define ORC_RESTRICT __restrict__ +#else +#define ORC_RESTRICT +#endif +#endif + +#ifndef ORC_INTERNAL +#if defined(__SUNPRO_C) && (__SUNPRO_C >= 0x590) +#define ORC_INTERNAL __attribute__((visibility("hidden"))) +#elif defined(__SUNPRO_C) && (__SUNPRO_C >= 0x550) +#define ORC_INTERNAL __hidden +#elif defined (__GNUC__) +#define ORC_INTERNAL __attribute__((visibility("hidden"))) +#else +#define ORC_INTERNAL +#endif +#endif + +void compositor_orc_splat_u32 (guint32 * ORC_RESTRICT d1, int p1, int n); +void compositor_orc_memcpy_u32 (guint32 * ORC_RESTRICT d1, const guint32 * ORC_RESTRICT s1, int n); +void compositor_orc_blend_u8 (guint8 * ORC_RESTRICT d1, int d1_stride, const guint8 * ORC_RESTRICT s1, int s1_stride, int p1, int n, int m); +void compositor_orc_blend_argb (guint8 * ORC_RESTRICT d1, int d1_stride, const guint8 * ORC_RESTRICT s1, int s1_stride, int p1, int n, int m); +void compositor_orc_blend_bgra (guint8 * ORC_RESTRICT d1, int d1_stride, const guint8 * ORC_RESTRICT s1, int s1_stride, int p1, int n, int m); +void compositor_orc_overlay_argb (guint8 * ORC_RESTRICT d1, int d1_stride, const guint8 * ORC_RESTRICT s1, int s1_stride, int p1, int n, int m); +void compositor_orc_overlay_bgra (guint8 * ORC_RESTRICT d1, int d1_stride, const guint8 * ORC_RESTRICT s1, int s1_stride, int p1, int n, int m); + +#ifdef __cplusplus +} +#endif + +#endif + diff --git a/gst/compositor/compositor.c b/gst/compositor/compositor.c new file mode 100644 index 0000000000..805b674543 --- /dev/null +++ b/gst/compositor/compositor.c @@ -0,0 +1,604 @@ +/* Video compositor plugin + * Copyright (C) 2004, 2008 Wim Taymans + * Copyright (C) 2010 Sebastian Dröge + * Copyright (C) 2014 Mathieu Duponchelle + * Copyright (C) 2014 Thibault Saunier + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +/** + * SECTION:element-compositor + * + * Compositor can accept AYUV, ARGB and BGRA video streams. For each of the requested + * sink pads it will compare the incoming geometry and framerate to define the + * output parameters. Indeed output video frames will have the geometry of the + * biggest incoming video stream and the framerate of the fastest incoming one. + * + * Compositor will do colorspace conversion. + * + * Individual parameters for each input stream can be configured on the + * #GstCompositorPad. + * + * + * Sample pipelines + * |[ + * gst-launch-1.0 \ + * videotestsrc pattern=1 ! \ + * video/x-raw,format=AYUV,framerate=\(fraction\)10/1,width=100,height=100 ! \ + * videobox border-alpha=0 top=-70 bottom=-70 right=-220 ! \ + * compositor name=comp sink_0::alpha=0.7 sink_1::alpha=0.5 ! \ + * videoconvert ! xvimagesink \ + * videotestsrc ! \ + * video/x-raw,format=AYUV,framerate=\(fraction\)5/1,width=320,height=240 ! comp. + * ]| A pipeline to demonstrate compositor used together with videobox. + * This should show a 320x240 pixels video test source with some transparency + * showing the background checker pattern. Another video test source with just + * the snow pattern of 100x100 pixels is overlayed on top of the first one on + * the left vertically centered with a small transparency showing the first + * video test source behind and the checker pattern under it. Note that the + * framerate of the output video is 10 frames per second. + * |[ + * gst-launch-1.0 videotestsrc pattern=1 ! \ + * video/x-raw, framerate=\(fraction\)10/1, width=100, height=100 ! \ + * compositor name=comp ! videoconvert ! ximagesink \ + * videotestsrc ! \ + * video/x-raw, framerate=\(fraction\)5/1, width=320, height=240 ! comp. + * ]| A pipeline to demostrate bgra comping. (This does not demonstrate alpha blending). + * |[ + * gst-launch-1.0 videotestsrc pattern=1 ! \ + * video/x-raw,format =I420, framerate=\(fraction\)10/1, width=100, height=100 ! \ + * compositor name=comp ! videoconvert ! ximagesink \ + * videotestsrc ! \ + * video/x-raw,format=I420, framerate=\(fraction\)5/1, width=320, height=240 ! comp. + * ]| A pipeline to test I420 + * |[ + * gst-launch-1.0 compositor name=comp sink_1::alpha=0.5 sink_1::xpos=50 sink_1::ypos=50 ! \ + * videoconvert ! ximagesink \ + * videotestsrc pattern=snow timestamp-offset=3000000000 ! \ + * "video/x-raw,format=AYUV,width=640,height=480,framerate=(fraction)30/1" ! \ + * timeoverlay ! queue2 ! comp. \ + * videotestsrc pattern=smpte ! \ + * "video/x-raw,format=AYUV,width=800,height=600,framerate=(fraction)10/1" ! \ + * timeoverlay ! queue2 ! comp. + * ]| A pipeline to demonstrate synchronized compositing (the second stream starts after 3 seconds) + * + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include + +#include "compositor.h" +#include "compositorpad.h" + +#ifdef DISABLE_ORC +#define orc_memset memset +#else +#include +#endif + +GST_DEBUG_CATEGORY_STATIC (gst_compositor_debug); +#define GST_CAT_DEFAULT gst_compositor_debug + +#define FORMATS " { AYUV, BGRA, ARGB, RGBA, ABGR, Y444, Y42B, YUY2, UYVY, "\ + " YVYU, I420, YV12, NV12, NV21, Y41B, RGB, BGR, xRGB, xBGR, "\ + " RGBx, BGRx } " + +static GstStaticPadTemplate src_factory = GST_STATIC_PAD_TEMPLATE ("src", + GST_PAD_SRC, + GST_PAD_ALWAYS, + GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE (FORMATS)) + ); + +static GstStaticPadTemplate sink_factory = GST_STATIC_PAD_TEMPLATE ("sink_%u", + GST_PAD_SINK, + GST_PAD_REQUEST, + GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE (FORMATS)) + ); + +#define DEFAULT_PAD_ZORDER 0 +#define DEFAULT_PAD_XPOS 0 +#define DEFAULT_PAD_YPOS 0 +#define DEFAULT_PAD_ALPHA 1.0 +enum +{ + PROP_PAD_0, + PROP_PAD_ZORDER, + PROP_PAD_XPOS, + PROP_PAD_YPOS, + PROP_PAD_ALPHA +}; + +G_DEFINE_TYPE (GstCompositorPad, gst_compositor_pad, + GST_TYPE_VIDEO_AGGREGATOR_PAD); + +static void +gst_compositor_pad_get_property (GObject * object, guint prop_id, + GValue * value, GParamSpec * pspec) +{ + GstCompositorPad *pad = GST_COMPOSITOR_PAD (object); + + switch (prop_id) { + case PROP_PAD_ZORDER: + g_value_set_uint (value, pad->zorder); + break; + case PROP_PAD_XPOS: + g_value_set_int (value, pad->xpos); + break; + case PROP_PAD_YPOS: + g_value_set_int (value, pad->ypos); + break; + case PROP_PAD_ALPHA: + g_value_set_double (value, pad->alpha); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static void +gst_compositor_pad_set_property (GObject * object, guint prop_id, + const GValue * value, GParamSpec * pspec) +{ + GstCompositorPad *pad = GST_COMPOSITOR_PAD (object); + + switch (prop_id) { + case PROP_PAD_XPOS: + pad->xpos = g_value_get_int (value); + break; + case PROP_PAD_YPOS: + pad->ypos = g_value_get_int (value); + break; + case PROP_PAD_ALPHA: + pad->alpha = g_value_get_double (value); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static void +gst_compositor_pad_class_init (GstCompositorPadClass * klass) +{ + GObjectClass *gobject_class = (GObjectClass *) klass; + + gobject_class->set_property = gst_compositor_pad_set_property; + gobject_class->get_property = gst_compositor_pad_get_property; + + g_object_class_install_property (gobject_class, PROP_PAD_XPOS, + g_param_spec_int ("xpos", "X Position", "X Position of the picture", + G_MININT, G_MAXINT, DEFAULT_PAD_XPOS, + G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); + g_object_class_install_property (gobject_class, PROP_PAD_YPOS, + g_param_spec_int ("ypos", "Y Position", "Y Position of the picture", + G_MININT, G_MAXINT, DEFAULT_PAD_YPOS, + G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); + g_object_class_install_property (gobject_class, PROP_PAD_ALPHA, + g_param_spec_double ("alpha", "Alpha", "Alpha of the picture", 0.0, 1.0, + DEFAULT_PAD_ALPHA, + G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); +} + +static void +gst_compositor_pad_init (GstCompositorPad * compo_pad) +{ + compo_pad->xpos = DEFAULT_PAD_XPOS; + compo_pad->ypos = DEFAULT_PAD_YPOS; + compo_pad->alpha = DEFAULT_PAD_ALPHA; +} + + +/* GstCompositor */ +#define DEFAULT_BACKGROUND COMPOSITOR_BACKGROUND_CHECKER +enum +{ + PROP_0, + PROP_BACKGROUND +}; + +#define GST_TYPE_COMPOSITOR_BACKGROUND (gst_compositor_background_get_type()) +static GType +gst_compositor_background_get_type (void) +{ + static GType compositor_background_type = 0; + + static const GEnumValue compositor_background[] = { + {COMPOSITOR_BACKGROUND_CHECKER, "Checker pattern", "checker"}, + {COMPOSITOR_BACKGROUND_BLACK, "Black", "black"}, + {COMPOSITOR_BACKGROUND_WHITE, "White", "white"}, + {COMPOSITOR_BACKGROUND_TRANSPARENT, + "Transparent Background to enable further compositing", "transparent"}, + {0, NULL, NULL}, + }; + + if (!compositor_background_type) { + compositor_background_type = + g_enum_register_static ("GstCompositorBackground", + compositor_background); + } + return compositor_background_type; +} + +static void +gst_compositor_get_property (GObject * object, + guint prop_id, GValue * value, GParamSpec * pspec) +{ + GstCompositor *self = GST_COMPOSITOR (object); + + switch (prop_id) { + case PROP_BACKGROUND: + g_value_set_enum (value, self->background); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static void +gst_compositor_set_property (GObject * object, + guint prop_id, const GValue * value, GParamSpec * pspec) +{ + GstCompositor *self = GST_COMPOSITOR (object); + + switch (prop_id) { + case PROP_BACKGROUND: + self->background = g_value_get_enum (value); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +#define gst_compositor_parent_class parent_class +G_DEFINE_TYPE (GstCompositor, gst_compositor, GST_TYPE_VIDEO_AGGREGATOR); + +static gboolean +set_functions (GstCompositor * self, GstVideoInfo * info) +{ + gboolean ret = FALSE; + + self->blend = NULL; + self->overlay = NULL; + self->fill_checker = NULL; + self->fill_color = NULL; + + switch (GST_VIDEO_INFO_FORMAT (info)) { + case GST_VIDEO_FORMAT_AYUV: + self->blend = gst_compositor_blend_ayuv; + self->overlay = gst_compositor_overlay_ayuv; + self->fill_checker = gst_compositor_fill_checker_ayuv; + self->fill_color = gst_compositor_fill_color_ayuv; + ret = TRUE; + break; + case GST_VIDEO_FORMAT_ARGB: + self->blend = gst_compositor_blend_argb; + self->overlay = gst_compositor_overlay_argb; + self->fill_checker = gst_compositor_fill_checker_argb; + self->fill_color = gst_compositor_fill_color_argb; + ret = TRUE; + break; + case GST_VIDEO_FORMAT_BGRA: + self->blend = gst_compositor_blend_bgra; + self->overlay = gst_compositor_overlay_bgra; + self->fill_checker = gst_compositor_fill_checker_bgra; + self->fill_color = gst_compositor_fill_color_bgra; + ret = TRUE; + break; + case GST_VIDEO_FORMAT_ABGR: + self->blend = gst_compositor_blend_abgr; + self->overlay = gst_compositor_overlay_abgr; + self->fill_checker = gst_compositor_fill_checker_abgr; + self->fill_color = gst_compositor_fill_color_abgr; + ret = TRUE; + break; + case GST_VIDEO_FORMAT_RGBA: + self->blend = gst_compositor_blend_rgba; + self->overlay = gst_compositor_overlay_rgba; + self->fill_checker = gst_compositor_fill_checker_rgba; + self->fill_color = gst_compositor_fill_color_rgba; + ret = TRUE; + break; + case GST_VIDEO_FORMAT_Y444: + self->blend = gst_compositor_blend_y444; + self->overlay = self->blend; + self->fill_checker = gst_compositor_fill_checker_y444; + self->fill_color = gst_compositor_fill_color_y444; + ret = TRUE; + break; + case GST_VIDEO_FORMAT_Y42B: + self->blend = gst_compositor_blend_y42b; + self->overlay = self->blend; + self->fill_checker = gst_compositor_fill_checker_y42b; + self->fill_color = gst_compositor_fill_color_y42b; + ret = TRUE; + break; + case GST_VIDEO_FORMAT_YUY2: + self->blend = gst_compositor_blend_yuy2; + self->overlay = self->blend; + self->fill_checker = gst_compositor_fill_checker_yuy2; + self->fill_color = gst_compositor_fill_color_yuy2; + ret = TRUE; + break; + case GST_VIDEO_FORMAT_UYVY: + self->blend = gst_compositor_blend_uyvy; + self->overlay = self->blend; + self->fill_checker = gst_compositor_fill_checker_uyvy; + self->fill_color = gst_compositor_fill_color_uyvy; + ret = TRUE; + break; + case GST_VIDEO_FORMAT_YVYU: + self->blend = gst_compositor_blend_yvyu; + self->overlay = self->blend; + self->fill_checker = gst_compositor_fill_checker_yvyu; + self->fill_color = gst_compositor_fill_color_yvyu; + ret = TRUE; + break; + case GST_VIDEO_FORMAT_I420: + self->blend = gst_compositor_blend_i420; + self->overlay = self->blend; + self->fill_checker = gst_compositor_fill_checker_i420; + self->fill_color = gst_compositor_fill_color_i420; + ret = TRUE; + break; + case GST_VIDEO_FORMAT_YV12: + self->blend = gst_compositor_blend_yv12; + self->overlay = self->blend; + self->fill_checker = gst_compositor_fill_checker_yv12; + self->fill_color = gst_compositor_fill_color_yv12; + ret = TRUE; + break; + case GST_VIDEO_FORMAT_NV12: + self->blend = gst_compositor_blend_nv12; + self->overlay = self->blend; + self->fill_checker = gst_compositor_fill_checker_nv12; + self->fill_color = gst_compositor_fill_color_nv12; + ret = TRUE; + break; + case GST_VIDEO_FORMAT_NV21: + self->blend = gst_compositor_blend_nv21; + self->overlay = self->blend; + self->fill_checker = gst_compositor_fill_checker_nv21; + self->fill_color = gst_compositor_fill_color_nv21; + ret = TRUE; + break; + case GST_VIDEO_FORMAT_Y41B: + self->blend = gst_compositor_blend_y41b; + self->overlay = self->blend; + self->fill_checker = gst_compositor_fill_checker_y41b; + self->fill_color = gst_compositor_fill_color_y41b; + ret = TRUE; + break; + case GST_VIDEO_FORMAT_RGB: + self->blend = gst_compositor_blend_rgb; + self->overlay = self->blend; + self->fill_checker = gst_compositor_fill_checker_rgb; + self->fill_color = gst_compositor_fill_color_rgb; + ret = TRUE; + break; + case GST_VIDEO_FORMAT_BGR: + self->blend = gst_compositor_blend_bgr; + self->overlay = self->blend; + self->fill_checker = gst_compositor_fill_checker_bgr; + self->fill_color = gst_compositor_fill_color_bgr; + ret = TRUE; + break; + case GST_VIDEO_FORMAT_xRGB: + self->blend = gst_compositor_blend_xrgb; + self->overlay = self->blend; + self->fill_checker = gst_compositor_fill_checker_xrgb; + self->fill_color = gst_compositor_fill_color_xrgb; + ret = TRUE; + break; + case GST_VIDEO_FORMAT_xBGR: + self->blend = gst_compositor_blend_xbgr; + self->overlay = self->blend; + self->fill_checker = gst_compositor_fill_checker_xbgr; + self->fill_color = gst_compositor_fill_color_xbgr; + ret = TRUE; + break; + case GST_VIDEO_FORMAT_RGBx: + self->blend = gst_compositor_blend_rgbx; + self->overlay = self->blend; + self->fill_checker = gst_compositor_fill_checker_rgbx; + self->fill_color = gst_compositor_fill_color_rgbx; + ret = TRUE; + break; + case GST_VIDEO_FORMAT_BGRx: + self->blend = gst_compositor_blend_bgrx; + self->overlay = self->blend; + self->fill_checker = gst_compositor_fill_checker_bgrx; + self->fill_color = gst_compositor_fill_color_bgrx; + ret = TRUE; + break; + default: + break; + } + + return ret; +} + +static gboolean +_update_info (GstVideoAggregator * vagg, GstVideoInfo * info) +{ + GList *l; + gint best_width = -1, best_height = -1; + gboolean ret = FALSE; + + GST_OBJECT_LOCK (vagg); + for (l = GST_ELEMENT (vagg)->sinkpads; l; l = l->next) { + GstVideoAggregatorPad *vaggpad = l->data; + GstCompositorPad *compositor_pad = GST_COMPOSITOR_PAD (vaggpad); + gint this_width, this_height; + gint width, height; + + width = GST_VIDEO_INFO_WIDTH (&vaggpad->info); + height = GST_VIDEO_INFO_HEIGHT (&vaggpad->info); + + if (width == 0 || height == 0) + continue; + + this_width = width + MAX (compositor_pad->xpos, 0); + this_height = height + MAX (compositor_pad->ypos, 0); + + if (best_width < this_width) + best_width = this_width; + if (best_height < this_height) + best_height = this_height; + } + GST_OBJECT_UNLOCK (vagg); + + if (best_width > 0 && best_height > 0) { + gst_video_info_set_format (info, GST_VIDEO_INFO_FORMAT (info), + best_width, best_height); + ret = set_functions (GST_COMPOSITOR (vagg), info); + } + + return ret; +} + +static GstFlowReturn +gst_compositor_aggregate_frames (GstVideoAggregator * vagg, GstBuffer * outbuf) +{ + GList *l; + GstCompositor *self = GST_COMPOSITOR (vagg); + BlendFunction composite; + GstVideoFrame out_frame, *outframe; + + if (!gst_video_frame_map (&out_frame, &vagg->info, outbuf, GST_MAP_WRITE)) { + + return GST_FLOW_ERROR; + } + + outframe = &out_frame; + /* default to blending */ + composite = self->blend; + switch (self->background) { + case COMPOSITOR_BACKGROUND_CHECKER: + self->fill_checker (outframe); + break; + case COMPOSITOR_BACKGROUND_BLACK: + self->fill_color (outframe, 16, 128, 128); + break; + case COMPOSITOR_BACKGROUND_WHITE: + self->fill_color (outframe, 240, 128, 128); + break; + case COMPOSITOR_BACKGROUND_TRANSPARENT: + { + guint i, plane, num_planes, height; + + num_planes = GST_VIDEO_FRAME_N_PLANES (outframe); + for (plane = 0; plane < num_planes; ++plane) { + guint8 *pdata; + gsize rowsize, plane_stride; + + pdata = GST_VIDEO_FRAME_PLANE_DATA (outframe, plane); + plane_stride = GST_VIDEO_FRAME_PLANE_STRIDE (outframe, plane); + rowsize = GST_VIDEO_FRAME_COMP_WIDTH (outframe, plane) + * GST_VIDEO_FRAME_COMP_PSTRIDE (outframe, plane); + height = GST_VIDEO_FRAME_COMP_HEIGHT (outframe, plane); + for (i = 0; i < height; ++i) { + memset (pdata, 0, rowsize); + pdata += plane_stride; + } + } + + /* use overlay to keep background transparent */ + composite = self->overlay; + break; + } + } + + GST_OBJECT_LOCK (vagg); + for (l = GST_ELEMENT (vagg)->sinkpads; l; l = l->next) { + GstVideoAggregatorPad *pad = l->data; + GstCompositorPad *compo_pad = GST_COMPOSITOR_PAD (pad); + + if (pad->aggregated_frame != NULL) { + composite (pad->aggregated_frame, compo_pad->xpos, compo_pad->ypos, + compo_pad->alpha, outframe); + } + } + GST_OBJECT_UNLOCK (vagg); + + gst_video_frame_unmap (outframe); + + return GST_FLOW_OK; +} + +/* GObject boilerplate */ +static void +gst_compositor_class_init (GstCompositorClass * klass) +{ + GObjectClass *gobject_class = (GObjectClass *) klass; + GstElementClass *gstelement_class = (GstElementClass *) klass; + GstVideoAggregatorClass *videoaggregator_class = + (GstVideoAggregatorClass *) klass; + GstAggregatorClass *agg_class = (GstAggregatorClass *) klass; + + gobject_class->get_property = gst_compositor_get_property; + gobject_class->set_property = gst_compositor_set_property; + + agg_class->sinkpads_type = GST_TYPE_COMPOSITOR_PAD; + videoaggregator_class->update_info = _update_info; + videoaggregator_class->aggregate_frames = gst_compositor_aggregate_frames; + + g_object_class_install_property (gobject_class, PROP_BACKGROUND, + g_param_spec_enum ("background", "Background", "Background type", + GST_TYPE_COMPOSITOR_BACKGROUND, + DEFAULT_BACKGROUND, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + + gst_element_class_add_pad_template (gstelement_class, + gst_static_pad_template_get (&src_factory)); + gst_element_class_add_pad_template (gstelement_class, + gst_static_pad_template_get (&sink_factory)); + + gst_element_class_set_static_metadata (gstelement_class, "Compositor", + "Filter/Editor/Video/Compositor", + "Composite multiple video streams", "Wim Taymans , " + "Sebastian Dröge "); +} + +static void +gst_compositor_init (GstCompositor * self) +{ + self->background = DEFAULT_BACKGROUND; + /* initialize variables */ +} + +/* Element registration */ +static gboolean +plugin_init (GstPlugin * plugin) +{ + GST_DEBUG_CATEGORY_INIT (gst_compositor_debug, "compositor", 0, "compositor"); + + gst_compositor_init_blend (); + + return gst_element_register (plugin, "compositor", GST_RANK_PRIMARY + 1, + GST_TYPE_COMPOSITOR); +} + +GST_PLUGIN_DEFINE (GST_VERSION_MAJOR, + GST_VERSION_MINOR, + compositor, + "Compositor", plugin_init, VERSION, GST_LICENSE, GST_PACKAGE_NAME, + GST_PACKAGE_ORIGIN) diff --git a/gst/compositor/compositor.h b/gst/compositor/compositor.h new file mode 100644 index 0000000000..a6b4d9fce1 --- /dev/null +++ b/gst/compositor/compositor.h @@ -0,0 +1,86 @@ +/* Generic video compositor plugin + * Copyright (C) 2008 Wim Taymans + * Copyright (C) 2010 Sebastian Dröge + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifndef __GST_COMPOSITOR_H__ +#define __GST_COMPOSITOR_H__ + +#include +#include +#include + +#include "blend.h" + +G_BEGIN_DECLS + +#define GST_TYPE_COMPOSITOR (gst_compositor_get_type()) +#define GST_COMPOSITOR(obj) \ + (G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_COMPOSITOR, GstCompositor)) +#define GST_COMPOSITOR_CLASS(klass) \ + (G_TYPE_CHECK_CLASS_CAST((klass),GST_TYPE_COMPOSITOR, GstCompositorClass)) +#define GST_IS_COMPOSITOR(obj) \ + (G_TYPE_CHECK_INSTANCE_TYPE((obj),GST_TYPE_COMPOSITOR)) +#define GST_IS_COMPOSITOR_CLASS(klass) \ + (G_TYPE_CHECK_CLASS_TYPE((klass),GST_TYPE_COMPOSITOR)) + +typedef struct _GstCompositor GstCompositor; +typedef struct _GstCompositorClass GstCompositorClass; + +/** + * GstcompositorBackground: + * @COMPOSITOR_BACKGROUND_CHECKER: checker pattern background + * @COMPOSITOR_BACKGROUND_BLACK: solid color black background + * @COMPOSITOR_BACKGROUND_WHITE: solid color white background + * @COMPOSITOR_BACKGROUND_TRANSPARENT: background is left transparent and layers are composited using "A OVER B" composition rules. This is only applicable to AYUV and ARGB (and variants) as it preserves the alpha channel and allows for further mixing. + * + * The different backgrounds compositor can blend over. + */ +typedef enum +{ + COMPOSITOR_BACKGROUND_CHECKER, + COMPOSITOR_BACKGROUND_BLACK, + COMPOSITOR_BACKGROUND_WHITE, + COMPOSITOR_BACKGROUND_TRANSPARENT, +} +GstCompositorBackground; + +/** + * GstCompositor: + * + * The opaque #GstCompositor structure. + */ +struct _GstCompositor +{ + GstVideoAggregator videoaggregator; + GstCompositorBackground background; + + BlendFunction blend, overlay; + FillCheckerFunction fill_checker; + FillColorFunction fill_color; +}; + +struct _GstCompositorClass +{ + GstVideoAggregatorClass parent_class; +}; + +GType gst_compositor_get_type (void); + +G_END_DECLS +#endif /* __GST_COMPOSITOR_H__ */ diff --git a/gst/compositor/compositororc-dist.c b/gst/compositor/compositororc-dist.c new file mode 100644 index 0000000000..04d97dd243 --- /dev/null +++ b/gst/compositor/compositororc-dist.c @@ -0,0 +1,2400 @@ + +/* autogenerated from compositororc.orc */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif +#include + +#ifndef _ORC_INTEGER_TYPEDEFS_ +#define _ORC_INTEGER_TYPEDEFS_ +#if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L +#include +typedef int8_t orc_int8; +typedef int16_t orc_int16; +typedef int32_t orc_int32; +typedef int64_t orc_int64; +typedef uint8_t orc_uint8; +typedef uint16_t orc_uint16; +typedef uint32_t orc_uint32; +typedef uint64_t orc_uint64; +#define ORC_UINT64_C(x) UINT64_C(x) +#elif defined(_MSC_VER) +typedef signed __int8 orc_int8; +typedef signed __int16 orc_int16; +typedef signed __int32 orc_int32; +typedef signed __int64 orc_int64; +typedef unsigned __int8 orc_uint8; +typedef unsigned __int16 orc_uint16; +typedef unsigned __int32 orc_uint32; +typedef unsigned __int64 orc_uint64; +#define ORC_UINT64_C(x) (x##Ui64) +#define inline __inline +#else +#include +typedef signed char orc_int8; +typedef short orc_int16; +typedef int orc_int32; +typedef unsigned char orc_uint8; +typedef unsigned short orc_uint16; +typedef unsigned int orc_uint32; +#if INT_MAX == LONG_MAX +typedef long long orc_int64; +typedef unsigned long long orc_uint64; +#define ORC_UINT64_C(x) (x##ULL) +#else +typedef long orc_int64; +typedef unsigned long orc_uint64; +#define ORC_UINT64_C(x) (x##UL) +#endif +#endif +typedef union +{ + orc_int16 i; + orc_int8 x2[2]; +} orc_union16; +typedef union +{ + orc_int32 i; + float f; + orc_int16 x2[2]; + orc_int8 x4[4]; +} orc_union32; +typedef union +{ + orc_int64 i; + double f; + orc_int32 x2[2]; + float x2f[2]; + orc_int16 x4[4]; +} orc_union64; +#endif +#ifndef ORC_RESTRICT +#if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L +#define ORC_RESTRICT restrict +#elif defined(__GNUC__) && __GNUC__ >= 4 +#define ORC_RESTRICT __restrict__ +#else +#define ORC_RESTRICT +#endif +#endif + +#ifndef ORC_INTERNAL +#if defined(__SUNPRO_C) && (__SUNPRO_C >= 0x590) +#define ORC_INTERNAL __attribute__((visibility("hidden"))) +#elif defined(__SUNPRO_C) && (__SUNPRO_C >= 0x550) +#define ORC_INTERNAL __hidden +#elif defined (__GNUC__) +#define ORC_INTERNAL __attribute__((visibility("hidden"))) +#else +#define ORC_INTERNAL +#endif +#endif + + +#ifndef DISABLE_ORC +#include +#endif +void compositor_orc_splat_u32 (guint32 * ORC_RESTRICT d1, int p1, int n); +void compositor_orc_memcpy_u32 (guint32 * ORC_RESTRICT d1, + const guint32 * ORC_RESTRICT s1, int n); +void compositor_orc_blend_u8 (guint8 * ORC_RESTRICT d1, int d1_stride, + const guint8 * ORC_RESTRICT s1, int s1_stride, int p1, int n, int m); +void compositor_orc_blend_argb (guint8 * ORC_RESTRICT d1, int d1_stride, + const guint8 * ORC_RESTRICT s1, int s1_stride, int p1, int n, int m); +void compositor_orc_blend_bgra (guint8 * ORC_RESTRICT d1, int d1_stride, + const guint8 * ORC_RESTRICT s1, int s1_stride, int p1, int n, int m); +void compositor_orc_overlay_argb (guint8 * ORC_RESTRICT d1, int d1_stride, + const guint8 * ORC_RESTRICT s1, int s1_stride, int p1, int n, int m); +void compositor_orc_overlay_bgra (guint8 * ORC_RESTRICT d1, int d1_stride, + const guint8 * ORC_RESTRICT s1, int s1_stride, int p1, int n, int m); + + +/* begin Orc C target preamble */ +#define ORC_CLAMP(x,a,b) ((x)<(a) ? (a) : ((x)>(b) ? (b) : (x))) +#define ORC_ABS(a) ((a)<0 ? -(a) : (a)) +#define ORC_MIN(a,b) ((a)<(b) ? (a) : (b)) +#define ORC_MAX(a,b) ((a)>(b) ? (a) : (b)) +#define ORC_SB_MAX 127 +#define ORC_SB_MIN (-1-ORC_SB_MAX) +#define ORC_UB_MAX 255 +#define ORC_UB_MIN 0 +#define ORC_SW_MAX 32767 +#define ORC_SW_MIN (-1-ORC_SW_MAX) +#define ORC_UW_MAX 65535 +#define ORC_UW_MIN 0 +#define ORC_SL_MAX 2147483647 +#define ORC_SL_MIN (-1-ORC_SL_MAX) +#define ORC_UL_MAX 4294967295U +#define ORC_UL_MIN 0 +#define ORC_CLAMP_SB(x) ORC_CLAMP(x,ORC_SB_MIN,ORC_SB_MAX) +#define ORC_CLAMP_UB(x) ORC_CLAMP(x,ORC_UB_MIN,ORC_UB_MAX) +#define ORC_CLAMP_SW(x) ORC_CLAMP(x,ORC_SW_MIN,ORC_SW_MAX) +#define ORC_CLAMP_UW(x) ORC_CLAMP(x,ORC_UW_MIN,ORC_UW_MAX) +#define ORC_CLAMP_SL(x) ORC_CLAMP(x,ORC_SL_MIN,ORC_SL_MAX) +#define ORC_CLAMP_UL(x) ORC_CLAMP(x,ORC_UL_MIN,ORC_UL_MAX) +#define ORC_SWAP_W(x) ((((x)&0xff)<<8) | (((x)&0xff00)>>8)) +#define ORC_SWAP_L(x) ((((x)&0xff)<<24) | (((x)&0xff00)<<8) | (((x)&0xff0000)>>8) | (((x)&0xff000000)>>24)) +#define ORC_SWAP_Q(x) ((((x)&ORC_UINT64_C(0xff))<<56) | (((x)&ORC_UINT64_C(0xff00))<<40) | (((x)&ORC_UINT64_C(0xff0000))<<24) | (((x)&ORC_UINT64_C(0xff000000))<<8) | (((x)&ORC_UINT64_C(0xff00000000))>>8) | (((x)&ORC_UINT64_C(0xff0000000000))>>24) | (((x)&ORC_UINT64_C(0xff000000000000))>>40) | (((x)&ORC_UINT64_C(0xff00000000000000))>>56)) +#define ORC_PTR_OFFSET(ptr,offset) ((void *)(((unsigned char *)(ptr)) + (offset))) +#define ORC_DENORMAL(x) ((x) & ((((x)&0x7f800000) == 0) ? 0xff800000 : 0xffffffff)) +#define ORC_ISNAN(x) ((((x)&0x7f800000) == 0x7f800000) && (((x)&0x007fffff) != 0)) +#define ORC_DENORMAL_DOUBLE(x) ((x) & ((((x)&ORC_UINT64_C(0x7ff0000000000000)) == 0) ? ORC_UINT64_C(0xfff0000000000000) : ORC_UINT64_C(0xffffffffffffffff))) +#define ORC_ISNAN_DOUBLE(x) ((((x)&ORC_UINT64_C(0x7ff0000000000000)) == ORC_UINT64_C(0x7ff0000000000000)) && (((x)&ORC_UINT64_C(0x000fffffffffffff)) != 0)) +#ifndef ORC_RESTRICT +#if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L +#define ORC_RESTRICT restrict +#elif defined(__GNUC__) && __GNUC__ >= 4 +#define ORC_RESTRICT __restrict__ +#else +#define ORC_RESTRICT +#endif +#endif +/* end Orc C target preamble */ + + + +/* compositor_orc_splat_u32 */ +#ifdef DISABLE_ORC +void +compositor_orc_splat_u32 (guint32 * ORC_RESTRICT d1, int p1, int n) +{ + int i; + orc_union32 *ORC_RESTRICT ptr0; + orc_union32 var32; + orc_union32 var33; + + ptr0 = (orc_union32 *) d1; + + /* 0: loadpl */ + var32.i = p1; + + for (i = 0; i < n; i++) { + /* 1: copyl */ + var33.i = var32.i; + /* 2: storel */ + ptr0[i] = var33; + } + +} + +#else +static void +_backup_compositor_orc_splat_u32 (OrcExecutor * ORC_RESTRICT ex) +{ + int i; + int n = ex->n; + orc_union32 *ORC_RESTRICT ptr0; + orc_union32 var32; + orc_union32 var33; + + ptr0 = (orc_union32 *) ex->arrays[0]; + + /* 0: loadpl */ + var32.i = ex->params[24]; + + for (i = 0; i < n; i++) { + /* 1: copyl */ + var33.i = var32.i; + /* 2: storel */ + ptr0[i] = var33; + } + +} + +void +compositor_orc_splat_u32 (guint32 * ORC_RESTRICT d1, int p1, int n) +{ + OrcExecutor _ex, *ex = &_ex; + static volatile int p_inited = 0; + static OrcCode *c = 0; + void (*func) (OrcExecutor *); + + if (!p_inited) { + orc_once_mutex_lock (); + if (!p_inited) { + OrcProgram *p; + +#if 1 + static const orc_uint8 bc[] = { + 1, 9, 24, 99, 111, 109, 112, 111, 115, 105, 116, 111, 114, 95, 111, 114, + 99, 95, 115, 112, 108, 97, 116, 95, 117, 51, 50, 11, 4, 4, 16, 4, + 112, 0, 24, 2, 0, + }; + p = orc_program_new_from_static_bytecode (bc); + orc_program_set_backup_function (p, _backup_compositor_orc_splat_u32); +#else + p = orc_program_new (); + orc_program_set_name (p, "compositor_orc_splat_u32"); + orc_program_set_backup_function (p, _backup_compositor_orc_splat_u32); + orc_program_add_destination (p, 4, "d1"); + orc_program_add_parameter (p, 4, "p1"); + + orc_program_append_2 (p, "copyl", 0, ORC_VAR_D1, ORC_VAR_P1, ORC_VAR_D1, + ORC_VAR_D1); +#endif + + orc_program_compile (p); + c = orc_program_take_code (p); + orc_program_free (p); + } + p_inited = TRUE; + orc_once_mutex_unlock (); + } + ex->arrays[ORC_VAR_A2] = c; + ex->program = 0; + + ex->n = n; + ex->arrays[ORC_VAR_D1] = d1; + ex->params[ORC_VAR_P1] = p1; + + func = c->exec; + func (ex); +} +#endif + + +/* compositor_orc_memcpy_u32 */ +#ifdef DISABLE_ORC +void +compositor_orc_memcpy_u32 (guint32 * ORC_RESTRICT d1, + const guint32 * ORC_RESTRICT s1, int n) +{ + int i; + orc_union32 *ORC_RESTRICT ptr0; + const orc_union32 *ORC_RESTRICT ptr4; + orc_union32 var32; + orc_union32 var33; + + ptr0 = (orc_union32 *) d1; + ptr4 = (orc_union32 *) s1; + + + for (i = 0; i < n; i++) { + /* 0: loadl */ + var32 = ptr4[i]; + /* 1: copyl */ + var33.i = var32.i; + /* 2: storel */ + ptr0[i] = var33; + } + +} + +#else +static void +_backup_compositor_orc_memcpy_u32 (OrcExecutor * ORC_RESTRICT ex) +{ + int i; + int n = ex->n; + orc_union32 *ORC_RESTRICT ptr0; + const orc_union32 *ORC_RESTRICT ptr4; + orc_union32 var32; + orc_union32 var33; + + ptr0 = (orc_union32 *) ex->arrays[0]; + ptr4 = (orc_union32 *) ex->arrays[4]; + + + for (i = 0; i < n; i++) { + /* 0: loadl */ + var32 = ptr4[i]; + /* 1: copyl */ + var33.i = var32.i; + /* 2: storel */ + ptr0[i] = var33; + } + +} + +void +compositor_orc_memcpy_u32 (guint32 * ORC_RESTRICT d1, + const guint32 * ORC_RESTRICT s1, int n) +{ + OrcExecutor _ex, *ex = &_ex; + static volatile int p_inited = 0; + static OrcCode *c = 0; + void (*func) (OrcExecutor *); + + if (!p_inited) { + orc_once_mutex_lock (); + if (!p_inited) { + OrcProgram *p; + +#if 1 + static const orc_uint8 bc[] = { + 1, 9, 25, 99, 111, 109, 112, 111, 115, 105, 116, 111, 114, 95, 111, 114, + 99, 95, 109, 101, 109, 99, 112, 121, 95, 117, 51, 50, 11, 4, 4, 12, + 4, 4, 112, 0, 4, 2, 0, + }; + p = orc_program_new_from_static_bytecode (bc); + orc_program_set_backup_function (p, _backup_compositor_orc_memcpy_u32); +#else + p = orc_program_new (); + orc_program_set_name (p, "compositor_orc_memcpy_u32"); + orc_program_set_backup_function (p, _backup_compositor_orc_memcpy_u32); + orc_program_add_destination (p, 4, "d1"); + orc_program_add_source (p, 4, "s1"); + + orc_program_append_2 (p, "copyl", 0, ORC_VAR_D1, ORC_VAR_S1, ORC_VAR_D1, + ORC_VAR_D1); +#endif + + orc_program_compile (p); + c = orc_program_take_code (p); + orc_program_free (p); + } + p_inited = TRUE; + orc_once_mutex_unlock (); + } + ex->arrays[ORC_VAR_A2] = c; + ex->program = 0; + + ex->n = n; + ex->arrays[ORC_VAR_D1] = d1; + ex->arrays[ORC_VAR_S1] = (void *) s1; + + func = c->exec; + func (ex); +} +#endif + + +/* compositor_orc_blend_u8 */ +#ifdef DISABLE_ORC +void +compositor_orc_blend_u8 (guint8 * ORC_RESTRICT d1, int d1_stride, + const guint8 * ORC_RESTRICT s1, int s1_stride, int p1, int n, int m) +{ + int i; + int j; + orc_int8 *ORC_RESTRICT ptr0; + const orc_int8 *ORC_RESTRICT ptr4; + orc_int8 var34; + orc_int8 var35; + orc_union16 var36; + orc_int8 var37; + orc_union16 var38; + orc_union16 var39; + orc_union16 var40; + orc_union16 var41; + orc_union16 var42; + orc_union16 var43; + orc_union16 var44; + + for (j = 0; j < m; j++) { + ptr0 = ORC_PTR_OFFSET (d1, d1_stride * j); + ptr4 = ORC_PTR_OFFSET (s1, s1_stride * j); + + /* 5: loadpw */ + var36.i = p1; + + for (i = 0; i < n; i++) { + /* 0: loadb */ + var34 = ptr0[i]; + /* 1: convubw */ + var38.i = (orc_uint8) var34; + /* 2: loadb */ + var35 = ptr4[i]; + /* 3: convubw */ + var39.i = (orc_uint8) var35; + /* 4: subw */ + var40.i = var39.i - var38.i; + /* 6: mullw */ + var41.i = (var40.i * var36.i) & 0xffff; + /* 7: shlw */ + var42.i = var38.i << 8; + /* 8: addw */ + var43.i = var42.i + var41.i; + /* 9: shruw */ + var44.i = ((orc_uint16) var43.i) >> 8; + /* 10: convsuswb */ + var37 = ORC_CLAMP_UB (var44.i); + /* 11: storeb */ + ptr0[i] = var37; + } + } + +} + +#else +static void +_backup_compositor_orc_blend_u8 (OrcExecutor * ORC_RESTRICT ex) +{ + int i; + int j; + int n = ex->n; + int m = ex->params[ORC_VAR_A1]; + orc_int8 *ORC_RESTRICT ptr0; + const orc_int8 *ORC_RESTRICT ptr4; + orc_int8 var34; + orc_int8 var35; + orc_union16 var36; + orc_int8 var37; + orc_union16 var38; + orc_union16 var39; + orc_union16 var40; + orc_union16 var41; + orc_union16 var42; + orc_union16 var43; + orc_union16 var44; + + for (j = 0; j < m; j++) { + ptr0 = ORC_PTR_OFFSET (ex->arrays[0], ex->params[0] * j); + ptr4 = ORC_PTR_OFFSET (ex->arrays[4], ex->params[4] * j); + + /* 5: loadpw */ + var36.i = ex->params[24]; + + for (i = 0; i < n; i++) { + /* 0: loadb */ + var34 = ptr0[i]; + /* 1: convubw */ + var38.i = (orc_uint8) var34; + /* 2: loadb */ + var35 = ptr4[i]; + /* 3: convubw */ + var39.i = (orc_uint8) var35; + /* 4: subw */ + var40.i = var39.i - var38.i; + /* 6: mullw */ + var41.i = (var40.i * var36.i) & 0xffff; + /* 7: shlw */ + var42.i = var38.i << 8; + /* 8: addw */ + var43.i = var42.i + var41.i; + /* 9: shruw */ + var44.i = ((orc_uint16) var43.i) >> 8; + /* 10: convsuswb */ + var37 = ORC_CLAMP_UB (var44.i); + /* 11: storeb */ + ptr0[i] = var37; + } + } + +} + +void +compositor_orc_blend_u8 (guint8 * ORC_RESTRICT d1, int d1_stride, + const guint8 * ORC_RESTRICT s1, int s1_stride, int p1, int n, int m) +{ + OrcExecutor _ex, *ex = &_ex; + static volatile int p_inited = 0; + static OrcCode *c = 0; + void (*func) (OrcExecutor *); + + if (!p_inited) { + orc_once_mutex_lock (); + if (!p_inited) { + OrcProgram *p; + +#if 1 + static const orc_uint8 bc[] = { + 1, 7, 9, 23, 99, 111, 109, 112, 111, 115, 105, 116, 111, 114, 95, 111, + 114, 99, 95, 98, 108, 101, 110, 100, 95, 117, 56, 11, 1, 1, 12, 1, + 1, 14, 1, 8, 0, 0, 0, 16, 2, 20, 2, 20, 2, 150, 32, 0, + 150, 33, 4, 98, 33, 33, 32, 89, 33, 33, 24, 93, 32, 32, 16, 70, + 33, 32, 33, 95, 33, 33, 16, 160, 0, 33, 2, 0, + }; + p = orc_program_new_from_static_bytecode (bc); + orc_program_set_backup_function (p, _backup_compositor_orc_blend_u8); +#else + p = orc_program_new (); + orc_program_set_2d (p); + orc_program_set_name (p, "compositor_orc_blend_u8"); + orc_program_set_backup_function (p, _backup_compositor_orc_blend_u8); + orc_program_add_destination (p, 1, "d1"); + orc_program_add_source (p, 1, "s1"); + orc_program_add_constant (p, 1, 0x00000008, "c1"); + orc_program_add_parameter (p, 2, "p1"); + orc_program_add_temporary (p, 2, "t1"); + orc_program_add_temporary (p, 2, "t2"); + + orc_program_append_2 (p, "convubw", 0, ORC_VAR_T1, ORC_VAR_D1, ORC_VAR_D1, + ORC_VAR_D1); + orc_program_append_2 (p, "convubw", 0, ORC_VAR_T2, ORC_VAR_S1, ORC_VAR_D1, + ORC_VAR_D1); + orc_program_append_2 (p, "subw", 0, ORC_VAR_T2, ORC_VAR_T2, ORC_VAR_T1, + ORC_VAR_D1); + orc_program_append_2 (p, "mullw", 0, ORC_VAR_T2, ORC_VAR_T2, ORC_VAR_P1, + ORC_VAR_D1); + orc_program_append_2 (p, "shlw", 0, ORC_VAR_T1, ORC_VAR_T1, ORC_VAR_C1, + ORC_VAR_D1); + orc_program_append_2 (p, "addw", 0, ORC_VAR_T2, ORC_VAR_T1, ORC_VAR_T2, + ORC_VAR_D1); + orc_program_append_2 (p, "shruw", 0, ORC_VAR_T2, ORC_VAR_T2, ORC_VAR_C1, + ORC_VAR_D1); + orc_program_append_2 (p, "convsuswb", 0, ORC_VAR_D1, ORC_VAR_T2, + ORC_VAR_D1, ORC_VAR_D1); +#endif + + orc_program_compile (p); + c = orc_program_take_code (p); + orc_program_free (p); + } + p_inited = TRUE; + orc_once_mutex_unlock (); + } + ex->arrays[ORC_VAR_A2] = c; + ex->program = 0; + + ex->n = n; + ORC_EXECUTOR_M (ex) = m; + ex->arrays[ORC_VAR_D1] = d1; + ex->params[ORC_VAR_D1] = d1_stride; + ex->arrays[ORC_VAR_S1] = (void *) s1; + ex->params[ORC_VAR_S1] = s1_stride; + ex->params[ORC_VAR_P1] = p1; + + func = c->exec; + func (ex); +} +#endif + + +/* compositor_orc_blend_argb */ +#ifdef DISABLE_ORC +void +compositor_orc_blend_argb (guint8 * ORC_RESTRICT d1, int d1_stride, + const guint8 * ORC_RESTRICT s1, int s1_stride, int p1, int n, int m) +{ + int i; + int j; + orc_union32 *ORC_RESTRICT ptr0; + const orc_union32 *ORC_RESTRICT ptr4; + orc_union64 var39; +#if defined(__APPLE__) && __GNUC__ == 4 && __GNUC_MINOR__ == 2 && defined (__i386__) + volatile orc_union32 var40; +#else + orc_union32 var40; +#endif + orc_union32 var41; + orc_union16 var42; + orc_int8 var43; + orc_union32 var44; + orc_union64 var45; + orc_union64 var46; + orc_union64 var47; + orc_union64 var48; + orc_union32 var49; + orc_union64 var50; + orc_union64 var51; + orc_union64 var52; + orc_union64 var53; + orc_union64 var54; + orc_union32 var55; + orc_union32 var56; + + for (j = 0; j < m; j++) { + ptr0 = ORC_PTR_OFFSET (d1, d1_stride * j); + ptr4 = ORC_PTR_OFFSET (s1, s1_stride * j); + + /* 5: loadpw */ + var39.x4[0] = p1; + var39.x4[1] = p1; + var39.x4[2] = p1; + var39.x4[3] = p1; + /* 16: loadpl */ + var40.i = (int) 0x000000ff; /* 255 or 1.25987e-321f */ + + for (i = 0; i < n; i++) { + /* 0: loadl */ + var41 = ptr4[i]; + /* 1: convlw */ + var42.i = var41.i; + /* 2: convwb */ + var43 = var42.i; + /* 3: splatbl */ + var44.i = + ((var43 & 0xff) << 24) | ((var43 & 0xff) << 16) | ((var43 & 0xff) << + 8) | (var43 & 0xff); + /* 4: convubw */ + var45.x4[0] = (orc_uint8) var44.x4[0]; + var45.x4[1] = (orc_uint8) var44.x4[1]; + var45.x4[2] = (orc_uint8) var44.x4[2]; + var45.x4[3] = (orc_uint8) var44.x4[3]; + /* 6: mullw */ + var46.x4[0] = (var45.x4[0] * var39.x4[0]) & 0xffff; + var46.x4[1] = (var45.x4[1] * var39.x4[1]) & 0xffff; + var46.x4[2] = (var45.x4[2] * var39.x4[2]) & 0xffff; + var46.x4[3] = (var45.x4[3] * var39.x4[3]) & 0xffff; + /* 7: shruw */ + var47.x4[0] = ((orc_uint16) var46.x4[0]) >> 8; + var47.x4[1] = ((orc_uint16) var46.x4[1]) >> 8; + var47.x4[2] = ((orc_uint16) var46.x4[2]) >> 8; + var47.x4[3] = ((orc_uint16) var46.x4[3]) >> 8; + /* 8: convubw */ + var48.x4[0] = (orc_uint8) var41.x4[0]; + var48.x4[1] = (orc_uint8) var41.x4[1]; + var48.x4[2] = (orc_uint8) var41.x4[2]; + var48.x4[3] = (orc_uint8) var41.x4[3]; + /* 9: loadl */ + var49 = ptr0[i]; + /* 10: convubw */ + var50.x4[0] = (orc_uint8) var49.x4[0]; + var50.x4[1] = (orc_uint8) var49.x4[1]; + var50.x4[2] = (orc_uint8) var49.x4[2]; + var50.x4[3] = (orc_uint8) var49.x4[3]; + /* 11: subw */ + var51.x4[0] = var48.x4[0] - var50.x4[0]; + var51.x4[1] = var48.x4[1] - var50.x4[1]; + var51.x4[2] = var48.x4[2] - var50.x4[2]; + var51.x4[3] = var48.x4[3] - var50.x4[3]; + /* 12: mullw */ + var52.x4[0] = (var51.x4[0] * var47.x4[0]) & 0xffff; + var52.x4[1] = (var51.x4[1] * var47.x4[1]) & 0xffff; + var52.x4[2] = (var51.x4[2] * var47.x4[2]) & 0xffff; + var52.x4[3] = (var51.x4[3] * var47.x4[3]) & 0xffff; + /* 13: div255w */ + var53.x4[0] = + ((orc_uint16) (((orc_uint16) (var52.x4[0] + 128)) + + (((orc_uint16) (var52.x4[0] + 128)) >> 8))) >> 8; + var53.x4[1] = + ((orc_uint16) (((orc_uint16) (var52.x4[1] + 128)) + + (((orc_uint16) (var52.x4[1] + 128)) >> 8))) >> 8; + var53.x4[2] = + ((orc_uint16) (((orc_uint16) (var52.x4[2] + 128)) + + (((orc_uint16) (var52.x4[2] + 128)) >> 8))) >> 8; + var53.x4[3] = + ((orc_uint16) (((orc_uint16) (var52.x4[3] + 128)) + + (((orc_uint16) (var52.x4[3] + 128)) >> 8))) >> 8; + /* 14: addw */ + var54.x4[0] = var50.x4[0] + var53.x4[0]; + var54.x4[1] = var50.x4[1] + var53.x4[1]; + var54.x4[2] = var50.x4[2] + var53.x4[2]; + var54.x4[3] = var50.x4[3] + var53.x4[3]; + /* 15: convwb */ + var55.x4[0] = var54.x4[0]; + var55.x4[1] = var54.x4[1]; + var55.x4[2] = var54.x4[2]; + var55.x4[3] = var54.x4[3]; + /* 17: orl */ + var56.i = var55.i | var40.i; + /* 18: storel */ + ptr0[i] = var56; + } + } + +} + +#else +static void +_backup_compositor_orc_blend_argb (OrcExecutor * ORC_RESTRICT ex) +{ + int i; + int j; + int n = ex->n; + int m = ex->params[ORC_VAR_A1]; + orc_union32 *ORC_RESTRICT ptr0; + const orc_union32 *ORC_RESTRICT ptr4; + orc_union64 var39; +#if defined(__APPLE__) && __GNUC__ == 4 && __GNUC_MINOR__ == 2 && defined (__i386__) + volatile orc_union32 var40; +#else + orc_union32 var40; +#endif + orc_union32 var41; + orc_union16 var42; + orc_int8 var43; + orc_union32 var44; + orc_union64 var45; + orc_union64 var46; + orc_union64 var47; + orc_union64 var48; + orc_union32 var49; + orc_union64 var50; + orc_union64 var51; + orc_union64 var52; + orc_union64 var53; + orc_union64 var54; + orc_union32 var55; + orc_union32 var56; + + for (j = 0; j < m; j++) { + ptr0 = ORC_PTR_OFFSET (ex->arrays[0], ex->params[0] * j); + ptr4 = ORC_PTR_OFFSET (ex->arrays[4], ex->params[4] * j); + + /* 5: loadpw */ + var39.x4[0] = ex->params[24]; + var39.x4[1] = ex->params[24]; + var39.x4[2] = ex->params[24]; + var39.x4[3] = ex->params[24]; + /* 16: loadpl */ + var40.i = (int) 0x000000ff; /* 255 or 1.25987e-321f */ + + for (i = 0; i < n; i++) { + /* 0: loadl */ + var41 = ptr4[i]; + /* 1: convlw */ + var42.i = var41.i; + /* 2: convwb */ + var43 = var42.i; + /* 3: splatbl */ + var44.i = + ((var43 & 0xff) << 24) | ((var43 & 0xff) << 16) | ((var43 & 0xff) << + 8) | (var43 & 0xff); + /* 4: convubw */ + var45.x4[0] = (orc_uint8) var44.x4[0]; + var45.x4[1] = (orc_uint8) var44.x4[1]; + var45.x4[2] = (orc_uint8) var44.x4[2]; + var45.x4[3] = (orc_uint8) var44.x4[3]; + /* 6: mullw */ + var46.x4[0] = (var45.x4[0] * var39.x4[0]) & 0xffff; + var46.x4[1] = (var45.x4[1] * var39.x4[1]) & 0xffff; + var46.x4[2] = (var45.x4[2] * var39.x4[2]) & 0xffff; + var46.x4[3] = (var45.x4[3] * var39.x4[3]) & 0xffff; + /* 7: shruw */ + var47.x4[0] = ((orc_uint16) var46.x4[0]) >> 8; + var47.x4[1] = ((orc_uint16) var46.x4[1]) >> 8; + var47.x4[2] = ((orc_uint16) var46.x4[2]) >> 8; + var47.x4[3] = ((orc_uint16) var46.x4[3]) >> 8; + /* 8: convubw */ + var48.x4[0] = (orc_uint8) var41.x4[0]; + var48.x4[1] = (orc_uint8) var41.x4[1]; + var48.x4[2] = (orc_uint8) var41.x4[2]; + var48.x4[3] = (orc_uint8) var41.x4[3]; + /* 9: loadl */ + var49 = ptr0[i]; + /* 10: convubw */ + var50.x4[0] = (orc_uint8) var49.x4[0]; + var50.x4[1] = (orc_uint8) var49.x4[1]; + var50.x4[2] = (orc_uint8) var49.x4[2]; + var50.x4[3] = (orc_uint8) var49.x4[3]; + /* 11: subw */ + var51.x4[0] = var48.x4[0] - var50.x4[0]; + var51.x4[1] = var48.x4[1] - var50.x4[1]; + var51.x4[2] = var48.x4[2] - var50.x4[2]; + var51.x4[3] = var48.x4[3] - var50.x4[3]; + /* 12: mullw */ + var52.x4[0] = (var51.x4[0] * var47.x4[0]) & 0xffff; + var52.x4[1] = (var51.x4[1] * var47.x4[1]) & 0xffff; + var52.x4[2] = (var51.x4[2] * var47.x4[2]) & 0xffff; + var52.x4[3] = (var51.x4[3] * var47.x4[3]) & 0xffff; + /* 13: div255w */ + var53.x4[0] = + ((orc_uint16) (((orc_uint16) (var52.x4[0] + 128)) + + (((orc_uint16) (var52.x4[0] + 128)) >> 8))) >> 8; + var53.x4[1] = + ((orc_uint16) (((orc_uint16) (var52.x4[1] + 128)) + + (((orc_uint16) (var52.x4[1] + 128)) >> 8))) >> 8; + var53.x4[2] = + ((orc_uint16) (((orc_uint16) (var52.x4[2] + 128)) + + (((orc_uint16) (var52.x4[2] + 128)) >> 8))) >> 8; + var53.x4[3] = + ((orc_uint16) (((orc_uint16) (var52.x4[3] + 128)) + + (((orc_uint16) (var52.x4[3] + 128)) >> 8))) >> 8; + /* 14: addw */ + var54.x4[0] = var50.x4[0] + var53.x4[0]; + var54.x4[1] = var50.x4[1] + var53.x4[1]; + var54.x4[2] = var50.x4[2] + var53.x4[2]; + var54.x4[3] = var50.x4[3] + var53.x4[3]; + /* 15: convwb */ + var55.x4[0] = var54.x4[0]; + var55.x4[1] = var54.x4[1]; + var55.x4[2] = var54.x4[2]; + var55.x4[3] = var54.x4[3]; + /* 17: orl */ + var56.i = var55.i | var40.i; + /* 18: storel */ + ptr0[i] = var56; + } + } + +} + +void +compositor_orc_blend_argb (guint8 * ORC_RESTRICT d1, int d1_stride, + const guint8 * ORC_RESTRICT s1, int s1_stride, int p1, int n, int m) +{ + OrcExecutor _ex, *ex = &_ex; + static volatile int p_inited = 0; + static OrcCode *c = 0; + void (*func) (OrcExecutor *); + + if (!p_inited) { + orc_once_mutex_lock (); + if (!p_inited) { + OrcProgram *p; + +#if 1 + static const orc_uint8 bc[] = { + 1, 7, 9, 25, 99, 111, 109, 112, 111, 115, 105, 116, 111, 114, 95, 111, + 114, 99, 95, 98, 108, 101, 110, 100, 95, 97, 114, 103, 98, 11, 4, 4, + 12, 4, 4, 14, 4, 255, 0, 0, 0, 14, 4, 8, 0, 0, 0, 16, + 2, 20, 4, 20, 2, 20, 1, 20, 4, 20, 8, 20, 8, 20, 8, 113, + 32, 4, 163, 33, 32, 157, 34, 33, 152, 35, 34, 21, 2, 150, 38, 35, + 21, 2, 89, 38, 38, 24, 21, 2, 95, 38, 38, 17, 21, 2, 150, 37, + 32, 113, 32, 0, 21, 2, 150, 36, 32, 21, 2, 98, 37, 37, 36, 21, + 2, 89, 37, 37, 38, 21, 2, 80, 37, 37, 21, 2, 70, 36, 36, 37, + 21, 2, 157, 32, 36, 123, 32, 32, 16, 128, 0, 32, 2, 0, + }; + p = orc_program_new_from_static_bytecode (bc); + orc_program_set_backup_function (p, _backup_compositor_orc_blend_argb); +#else + p = orc_program_new (); + orc_program_set_2d (p); + orc_program_set_name (p, "compositor_orc_blend_argb"); + orc_program_set_backup_function (p, _backup_compositor_orc_blend_argb); + orc_program_add_destination (p, 4, "d1"); + orc_program_add_source (p, 4, "s1"); + orc_program_add_constant (p, 4, 0x000000ff, "c1"); + orc_program_add_constant (p, 4, 0x00000008, "c2"); + orc_program_add_parameter (p, 2, "p1"); + orc_program_add_temporary (p, 4, "t1"); + orc_program_add_temporary (p, 2, "t2"); + orc_program_add_temporary (p, 1, "t3"); + orc_program_add_temporary (p, 4, "t4"); + orc_program_add_temporary (p, 8, "t5"); + orc_program_add_temporary (p, 8, "t6"); + orc_program_add_temporary (p, 8, "t7"); + + orc_program_append_2 (p, "loadl", 0, ORC_VAR_T1, ORC_VAR_S1, ORC_VAR_D1, + ORC_VAR_D1); + orc_program_append_2 (p, "convlw", 0, ORC_VAR_T2, ORC_VAR_T1, ORC_VAR_D1, + ORC_VAR_D1); + orc_program_append_2 (p, "convwb", 0, ORC_VAR_T3, ORC_VAR_T2, ORC_VAR_D1, + ORC_VAR_D1); + orc_program_append_2 (p, "splatbl", 0, ORC_VAR_T4, ORC_VAR_T3, ORC_VAR_D1, + ORC_VAR_D1); + orc_program_append_2 (p, "convubw", 2, ORC_VAR_T7, ORC_VAR_T4, ORC_VAR_D1, + ORC_VAR_D1); + orc_program_append_2 (p, "mullw", 2, ORC_VAR_T7, ORC_VAR_T7, ORC_VAR_P1, + ORC_VAR_D1); + orc_program_append_2 (p, "shruw", 2, ORC_VAR_T7, ORC_VAR_T7, ORC_VAR_C2, + ORC_VAR_D1); + orc_program_append_2 (p, "convubw", 2, ORC_VAR_T6, ORC_VAR_T1, ORC_VAR_D1, + ORC_VAR_D1); + orc_program_append_2 (p, "loadl", 0, ORC_VAR_T1, ORC_VAR_D1, ORC_VAR_D1, + ORC_VAR_D1); + orc_program_append_2 (p, "convubw", 2, ORC_VAR_T5, ORC_VAR_T1, ORC_VAR_D1, + ORC_VAR_D1); + orc_program_append_2 (p, "subw", 2, ORC_VAR_T6, ORC_VAR_T6, ORC_VAR_T5, + ORC_VAR_D1); + orc_program_append_2 (p, "mullw", 2, ORC_VAR_T6, ORC_VAR_T6, ORC_VAR_T7, + ORC_VAR_D1); + orc_program_append_2 (p, "div255w", 2, ORC_VAR_T6, ORC_VAR_T6, ORC_VAR_D1, + ORC_VAR_D1); + orc_program_append_2 (p, "addw", 2, ORC_VAR_T5, ORC_VAR_T5, ORC_VAR_T6, + ORC_VAR_D1); + orc_program_append_2 (p, "convwb", 2, ORC_VAR_T1, ORC_VAR_T5, ORC_VAR_D1, + ORC_VAR_D1); + orc_program_append_2 (p, "orl", 0, ORC_VAR_T1, ORC_VAR_T1, ORC_VAR_C1, + ORC_VAR_D1); + orc_program_append_2 (p, "storel", 0, ORC_VAR_D1, ORC_VAR_T1, ORC_VAR_D1, + ORC_VAR_D1); +#endif + + orc_program_compile (p); + c = orc_program_take_code (p); + orc_program_free (p); + } + p_inited = TRUE; + orc_once_mutex_unlock (); + } + ex->arrays[ORC_VAR_A2] = c; + ex->program = 0; + + ex->n = n; + ORC_EXECUTOR_M (ex) = m; + ex->arrays[ORC_VAR_D1] = d1; + ex->params[ORC_VAR_D1] = d1_stride; + ex->arrays[ORC_VAR_S1] = (void *) s1; + ex->params[ORC_VAR_S1] = s1_stride; + ex->params[ORC_VAR_P1] = p1; + + func = c->exec; + func (ex); +} +#endif + + +/* compositor_orc_blend_bgra */ +#ifdef DISABLE_ORC +void +compositor_orc_blend_bgra (guint8 * ORC_RESTRICT d1, int d1_stride, + const guint8 * ORC_RESTRICT s1, int s1_stride, int p1, int n, int m) +{ + int i; + int j; + orc_union32 *ORC_RESTRICT ptr0; + const orc_union32 *ORC_RESTRICT ptr4; + orc_union64 var40; +#if defined(__APPLE__) && __GNUC__ == 4 && __GNUC_MINOR__ == 2 && defined (__i386__) + volatile orc_union32 var41; +#else + orc_union32 var41; +#endif + orc_union32 var42; + orc_union32 var43; + orc_union16 var44; + orc_int8 var45; + orc_union32 var46; + orc_union64 var47; + orc_union64 var48; + orc_union64 var49; + orc_union64 var50; + orc_union32 var51; + orc_union64 var52; + orc_union64 var53; + orc_union64 var54; + orc_union64 var55; + orc_union64 var56; + orc_union32 var57; + orc_union32 var58; + + for (j = 0; j < m; j++) { + ptr0 = ORC_PTR_OFFSET (d1, d1_stride * j); + ptr4 = ORC_PTR_OFFSET (s1, s1_stride * j); + + /* 6: loadpw */ + var40.x4[0] = p1; + var40.x4[1] = p1; + var40.x4[2] = p1; + var40.x4[3] = p1; + /* 17: loadpl */ + var41.i = (int) 0xff000000; /* -16777216 or 2.11371e-314f */ + + for (i = 0; i < n; i++) { + /* 0: loadl */ + var42 = ptr4[i]; + /* 1: shrul */ + var43.i = ((orc_uint32) var42.i) >> 24; + /* 2: convlw */ + var44.i = var43.i; + /* 3: convwb */ + var45 = var44.i; + /* 4: splatbl */ + var46.i = + ((var45 & 0xff) << 24) | ((var45 & 0xff) << 16) | ((var45 & 0xff) << + 8) | (var45 & 0xff); + /* 5: convubw */ + var47.x4[0] = (orc_uint8) var46.x4[0]; + var47.x4[1] = (orc_uint8) var46.x4[1]; + var47.x4[2] = (orc_uint8) var46.x4[2]; + var47.x4[3] = (orc_uint8) var46.x4[3]; + /* 7: mullw */ + var48.x4[0] = (var47.x4[0] * var40.x4[0]) & 0xffff; + var48.x4[1] = (var47.x4[1] * var40.x4[1]) & 0xffff; + var48.x4[2] = (var47.x4[2] * var40.x4[2]) & 0xffff; + var48.x4[3] = (var47.x4[3] * var40.x4[3]) & 0xffff; + /* 8: shruw */ + var49.x4[0] = ((orc_uint16) var48.x4[0]) >> 8; + var49.x4[1] = ((orc_uint16) var48.x4[1]) >> 8; + var49.x4[2] = ((orc_uint16) var48.x4[2]) >> 8; + var49.x4[3] = ((orc_uint16) var48.x4[3]) >> 8; + /* 9: convubw */ + var50.x4[0] = (orc_uint8) var42.x4[0]; + var50.x4[1] = (orc_uint8) var42.x4[1]; + var50.x4[2] = (orc_uint8) var42.x4[2]; + var50.x4[3] = (orc_uint8) var42.x4[3]; + /* 10: loadl */ + var51 = ptr0[i]; + /* 11: convubw */ + var52.x4[0] = (orc_uint8) var51.x4[0]; + var52.x4[1] = (orc_uint8) var51.x4[1]; + var52.x4[2] = (orc_uint8) var51.x4[2]; + var52.x4[3] = (orc_uint8) var51.x4[3]; + /* 12: subw */ + var53.x4[0] = var50.x4[0] - var52.x4[0]; + var53.x4[1] = var50.x4[1] - var52.x4[1]; + var53.x4[2] = var50.x4[2] - var52.x4[2]; + var53.x4[3] = var50.x4[3] - var52.x4[3]; + /* 13: mullw */ + var54.x4[0] = (var53.x4[0] * var49.x4[0]) & 0xffff; + var54.x4[1] = (var53.x4[1] * var49.x4[1]) & 0xffff; + var54.x4[2] = (var53.x4[2] * var49.x4[2]) & 0xffff; + var54.x4[3] = (var53.x4[3] * var49.x4[3]) & 0xffff; + /* 14: div255w */ + var55.x4[0] = + ((orc_uint16) (((orc_uint16) (var54.x4[0] + 128)) + + (((orc_uint16) (var54.x4[0] + 128)) >> 8))) >> 8; + var55.x4[1] = + ((orc_uint16) (((orc_uint16) (var54.x4[1] + 128)) + + (((orc_uint16) (var54.x4[1] + 128)) >> 8))) >> 8; + var55.x4[2] = + ((orc_uint16) (((orc_uint16) (var54.x4[2] + 128)) + + (((orc_uint16) (var54.x4[2] + 128)) >> 8))) >> 8; + var55.x4[3] = + ((orc_uint16) (((orc_uint16) (var54.x4[3] + 128)) + + (((orc_uint16) (var54.x4[3] + 128)) >> 8))) >> 8; + /* 15: addw */ + var56.x4[0] = var52.x4[0] + var55.x4[0]; + var56.x4[1] = var52.x4[1] + var55.x4[1]; + var56.x4[2] = var52.x4[2] + var55.x4[2]; + var56.x4[3] = var52.x4[3] + var55.x4[3]; + /* 16: convwb */ + var57.x4[0] = var56.x4[0]; + var57.x4[1] = var56.x4[1]; + var57.x4[2] = var56.x4[2]; + var57.x4[3] = var56.x4[3]; + /* 18: orl */ + var58.i = var57.i | var41.i; + /* 19: storel */ + ptr0[i] = var58; + } + } + +} + +#else +static void +_backup_compositor_orc_blend_bgra (OrcExecutor * ORC_RESTRICT ex) +{ + int i; + int j; + int n = ex->n; + int m = ex->params[ORC_VAR_A1]; + orc_union32 *ORC_RESTRICT ptr0; + const orc_union32 *ORC_RESTRICT ptr4; + orc_union64 var40; +#if defined(__APPLE__) && __GNUC__ == 4 && __GNUC_MINOR__ == 2 && defined (__i386__) + volatile orc_union32 var41; +#else + orc_union32 var41; +#endif + orc_union32 var42; + orc_union32 var43; + orc_union16 var44; + orc_int8 var45; + orc_union32 var46; + orc_union64 var47; + orc_union64 var48; + orc_union64 var49; + orc_union64 var50; + orc_union32 var51; + orc_union64 var52; + orc_union64 var53; + orc_union64 var54; + orc_union64 var55; + orc_union64 var56; + orc_union32 var57; + orc_union32 var58; + + for (j = 0; j < m; j++) { + ptr0 = ORC_PTR_OFFSET (ex->arrays[0], ex->params[0] * j); + ptr4 = ORC_PTR_OFFSET (ex->arrays[4], ex->params[4] * j); + + /* 6: loadpw */ + var40.x4[0] = ex->params[24]; + var40.x4[1] = ex->params[24]; + var40.x4[2] = ex->params[24]; + var40.x4[3] = ex->params[24]; + /* 17: loadpl */ + var41.i = (int) 0xff000000; /* -16777216 or 2.11371e-314f */ + + for (i = 0; i < n; i++) { + /* 0: loadl */ + var42 = ptr4[i]; + /* 1: shrul */ + var43.i = ((orc_uint32) var42.i) >> 24; + /* 2: convlw */ + var44.i = var43.i; + /* 3: convwb */ + var45 = var44.i; + /* 4: splatbl */ + var46.i = + ((var45 & 0xff) << 24) | ((var45 & 0xff) << 16) | ((var45 & 0xff) << + 8) | (var45 & 0xff); + /* 5: convubw */ + var47.x4[0] = (orc_uint8) var46.x4[0]; + var47.x4[1] = (orc_uint8) var46.x4[1]; + var47.x4[2] = (orc_uint8) var46.x4[2]; + var47.x4[3] = (orc_uint8) var46.x4[3]; + /* 7: mullw */ + var48.x4[0] = (var47.x4[0] * var40.x4[0]) & 0xffff; + var48.x4[1] = (var47.x4[1] * var40.x4[1]) & 0xffff; + var48.x4[2] = (var47.x4[2] * var40.x4[2]) & 0xffff; + var48.x4[3] = (var47.x4[3] * var40.x4[3]) & 0xffff; + /* 8: shruw */ + var49.x4[0] = ((orc_uint16) var48.x4[0]) >> 8; + var49.x4[1] = ((orc_uint16) var48.x4[1]) >> 8; + var49.x4[2] = ((orc_uint16) var48.x4[2]) >> 8; + var49.x4[3] = ((orc_uint16) var48.x4[3]) >> 8; + /* 9: convubw */ + var50.x4[0] = (orc_uint8) var42.x4[0]; + var50.x4[1] = (orc_uint8) var42.x4[1]; + var50.x4[2] = (orc_uint8) var42.x4[2]; + var50.x4[3] = (orc_uint8) var42.x4[3]; + /* 10: loadl */ + var51 = ptr0[i]; + /* 11: convubw */ + var52.x4[0] = (orc_uint8) var51.x4[0]; + var52.x4[1] = (orc_uint8) var51.x4[1]; + var52.x4[2] = (orc_uint8) var51.x4[2]; + var52.x4[3] = (orc_uint8) var51.x4[3]; + /* 12: subw */ + var53.x4[0] = var50.x4[0] - var52.x4[0]; + var53.x4[1] = var50.x4[1] - var52.x4[1]; + var53.x4[2] = var50.x4[2] - var52.x4[2]; + var53.x4[3] = var50.x4[3] - var52.x4[3]; + /* 13: mullw */ + var54.x4[0] = (var53.x4[0] * var49.x4[0]) & 0xffff; + var54.x4[1] = (var53.x4[1] * var49.x4[1]) & 0xffff; + var54.x4[2] = (var53.x4[2] * var49.x4[2]) & 0xffff; + var54.x4[3] = (var53.x4[3] * var49.x4[3]) & 0xffff; + /* 14: div255w */ + var55.x4[0] = + ((orc_uint16) (((orc_uint16) (var54.x4[0] + 128)) + + (((orc_uint16) (var54.x4[0] + 128)) >> 8))) >> 8; + var55.x4[1] = + ((orc_uint16) (((orc_uint16) (var54.x4[1] + 128)) + + (((orc_uint16) (var54.x4[1] + 128)) >> 8))) >> 8; + var55.x4[2] = + ((orc_uint16) (((orc_uint16) (var54.x4[2] + 128)) + + (((orc_uint16) (var54.x4[2] + 128)) >> 8))) >> 8; + var55.x4[3] = + ((orc_uint16) (((orc_uint16) (var54.x4[3] + 128)) + + (((orc_uint16) (var54.x4[3] + 128)) >> 8))) >> 8; + /* 15: addw */ + var56.x4[0] = var52.x4[0] + var55.x4[0]; + var56.x4[1] = var52.x4[1] + var55.x4[1]; + var56.x4[2] = var52.x4[2] + var55.x4[2]; + var56.x4[3] = var52.x4[3] + var55.x4[3]; + /* 16: convwb */ + var57.x4[0] = var56.x4[0]; + var57.x4[1] = var56.x4[1]; + var57.x4[2] = var56.x4[2]; + var57.x4[3] = var56.x4[3]; + /* 18: orl */ + var58.i = var57.i | var41.i; + /* 19: storel */ + ptr0[i] = var58; + } + } + +} + +void +compositor_orc_blend_bgra (guint8 * ORC_RESTRICT d1, int d1_stride, + const guint8 * ORC_RESTRICT s1, int s1_stride, int p1, int n, int m) +{ + OrcExecutor _ex, *ex = &_ex; + static volatile int p_inited = 0; + static OrcCode *c = 0; + void (*func) (OrcExecutor *); + + if (!p_inited) { + orc_once_mutex_lock (); + if (!p_inited) { + OrcProgram *p; + +#if 1 + static const orc_uint8 bc[] = { + 1, 7, 9, 25, 99, 111, 109, 112, 111, 115, 105, 116, 111, 114, 95, 111, + 114, 99, 95, 98, 108, 101, 110, 100, 95, 98, 103, 114, 97, 11, 4, 4, + 12, 4, 4, 14, 4, 0, 0, 0, 255, 14, 4, 24, 0, 0, 0, 14, + 4, 8, 0, 0, 0, 16, 2, 20, 4, 20, 4, 20, 2, 20, 1, 20, + 4, 20, 8, 20, 8, 20, 8, 113, 32, 4, 126, 33, 32, 17, 163, 34, + 33, 157, 35, 34, 152, 36, 35, 21, 2, 150, 39, 36, 21, 2, 89, 39, + 39, 24, 21, 2, 95, 39, 39, 18, 21, 2, 150, 38, 32, 113, 32, 0, + 21, 2, 150, 37, 32, 21, 2, 98, 38, 38, 37, 21, 2, 89, 38, 38, + 39, 21, 2, 80, 38, 38, 21, 2, 70, 37, 37, 38, 21, 2, 157, 32, + 37, 123, 32, 32, 16, 128, 0, 32, 2, 0, + }; + p = orc_program_new_from_static_bytecode (bc); + orc_program_set_backup_function (p, _backup_compositor_orc_blend_bgra); +#else + p = orc_program_new (); + orc_program_set_2d (p); + orc_program_set_name (p, "compositor_orc_blend_bgra"); + orc_program_set_backup_function (p, _backup_compositor_orc_blend_bgra); + orc_program_add_destination (p, 4, "d1"); + orc_program_add_source (p, 4, "s1"); + orc_program_add_constant (p, 4, 0xff000000, "c1"); + orc_program_add_constant (p, 4, 0x00000018, "c2"); + orc_program_add_constant (p, 4, 0x00000008, "c3"); + orc_program_add_parameter (p, 2, "p1"); + orc_program_add_temporary (p, 4, "t1"); + orc_program_add_temporary (p, 4, "t2"); + orc_program_add_temporary (p, 2, "t3"); + orc_program_add_temporary (p, 1, "t4"); + orc_program_add_temporary (p, 4, "t5"); + orc_program_add_temporary (p, 8, "t6"); + orc_program_add_temporary (p, 8, "t7"); + orc_program_add_temporary (p, 8, "t8"); + + orc_program_append_2 (p, "loadl", 0, ORC_VAR_T1, ORC_VAR_S1, ORC_VAR_D1, + ORC_VAR_D1); + orc_program_append_2 (p, "shrul", 0, ORC_VAR_T2, ORC_VAR_T1, ORC_VAR_C2, + ORC_VAR_D1); + orc_program_append_2 (p, "convlw", 0, ORC_VAR_T3, ORC_VAR_T2, ORC_VAR_D1, + ORC_VAR_D1); + orc_program_append_2 (p, "convwb", 0, ORC_VAR_T4, ORC_VAR_T3, ORC_VAR_D1, + ORC_VAR_D1); + orc_program_append_2 (p, "splatbl", 0, ORC_VAR_T5, ORC_VAR_T4, ORC_VAR_D1, + ORC_VAR_D1); + orc_program_append_2 (p, "convubw", 2, ORC_VAR_T8, ORC_VAR_T5, ORC_VAR_D1, + ORC_VAR_D1); + orc_program_append_2 (p, "mullw", 2, ORC_VAR_T8, ORC_VAR_T8, ORC_VAR_P1, + ORC_VAR_D1); + orc_program_append_2 (p, "shruw", 2, ORC_VAR_T8, ORC_VAR_T8, ORC_VAR_C3, + ORC_VAR_D1); + orc_program_append_2 (p, "convubw", 2, ORC_VAR_T7, ORC_VAR_T1, ORC_VAR_D1, + ORC_VAR_D1); + orc_program_append_2 (p, "loadl", 0, ORC_VAR_T1, ORC_VAR_D1, ORC_VAR_D1, + ORC_VAR_D1); + orc_program_append_2 (p, "convubw", 2, ORC_VAR_T6, ORC_VAR_T1, ORC_VAR_D1, + ORC_VAR_D1); + orc_program_append_2 (p, "subw", 2, ORC_VAR_T7, ORC_VAR_T7, ORC_VAR_T6, + ORC_VAR_D1); + orc_program_append_2 (p, "mullw", 2, ORC_VAR_T7, ORC_VAR_T7, ORC_VAR_T8, + ORC_VAR_D1); + orc_program_append_2 (p, "div255w", 2, ORC_VAR_T7, ORC_VAR_T7, ORC_VAR_D1, + ORC_VAR_D1); + orc_program_append_2 (p, "addw", 2, ORC_VAR_T6, ORC_VAR_T6, ORC_VAR_T7, + ORC_VAR_D1); + orc_program_append_2 (p, "convwb", 2, ORC_VAR_T1, ORC_VAR_T6, ORC_VAR_D1, + ORC_VAR_D1); + orc_program_append_2 (p, "orl", 0, ORC_VAR_T1, ORC_VAR_T1, ORC_VAR_C1, + ORC_VAR_D1); + orc_program_append_2 (p, "storel", 0, ORC_VAR_D1, ORC_VAR_T1, ORC_VAR_D1, + ORC_VAR_D1); +#endif + + orc_program_compile (p); + c = orc_program_take_code (p); + orc_program_free (p); + } + p_inited = TRUE; + orc_once_mutex_unlock (); + } + ex->arrays[ORC_VAR_A2] = c; + ex->program = 0; + + ex->n = n; + ORC_EXECUTOR_M (ex) = m; + ex->arrays[ORC_VAR_D1] = d1; + ex->params[ORC_VAR_D1] = d1_stride; + ex->arrays[ORC_VAR_S1] = (void *) s1; + ex->params[ORC_VAR_S1] = s1_stride; + ex->params[ORC_VAR_P1] = p1; + + func = c->exec; + func (ex); +} +#endif + + +/* compositor_orc_overlay_argb */ +#ifdef DISABLE_ORC +void +compositor_orc_overlay_argb (guint8 * ORC_RESTRICT d1, int d1_stride, + const guint8 * ORC_RESTRICT s1, int s1_stride, int p1, int n, int m) +{ + int i; + int j; + orc_union32 *ORC_RESTRICT ptr0; + const orc_union32 *ORC_RESTRICT ptr4; + orc_union64 var41; +#if defined(__APPLE__) && __GNUC__ == 4 && __GNUC_MINOR__ == 2 && defined (__i386__) + volatile orc_union32 var42; +#else + orc_union32 var42; +#endif +#if defined(__APPLE__) && __GNUC__ == 4 && __GNUC_MINOR__ == 2 && defined (__i386__) + volatile orc_union32 var43; +#else + orc_union32 var43; +#endif + orc_union32 var44; + orc_union16 var45; + orc_int8 var46; + orc_union32 var47; + orc_union64 var48; + orc_union64 var49; + orc_union64 var50; + orc_union64 var51; + orc_union64 var52; + orc_union32 var53; + orc_union64 var54; + orc_union64 var55; + orc_union32 var56; + orc_union16 var57; + orc_int8 var58; + orc_union32 var59; + orc_union64 var60; + orc_union64 var61; + orc_union64 var62; + orc_union64 var63; + orc_union64 var64; + orc_union64 var65; + orc_union64 var66; + orc_union64 var67; + orc_union32 var68; + orc_union32 var69; + orc_union32 var70; + orc_union32 var71; + orc_union32 var72; + + for (j = 0; j < m; j++) { + ptr0 = ORC_PTR_OFFSET (d1, d1_stride * j); + ptr4 = ORC_PTR_OFFSET (s1, s1_stride * j); + + /* 5: loadpw */ + var41.x4[0] = p1; + var41.x4[1] = p1; + var41.x4[2] = p1; + var41.x4[3] = p1; + /* 10: loadpl */ + var53.i = (int) 0xffffffff; /* -1 or 2.122e-314f */ + /* 26: loadpl */ + var42.i = (int) 0xffffff00; /* -256 or 2.122e-314f */ + /* 29: loadpl */ + var43.i = (int) 0x000000ff; /* 255 or 1.25987e-321f */ + + for (i = 0; i < n; i++) { + /* 0: loadl */ + var44 = ptr4[i]; + /* 1: convlw */ + var45.i = var44.i; + /* 2: convwb */ + var46 = var45.i; + /* 3: splatbl */ + var47.i = + ((var46 & 0xff) << 24) | ((var46 & 0xff) << 16) | ((var46 & 0xff) << + 8) | (var46 & 0xff); + /* 4: convubw */ + var48.x4[0] = (orc_uint8) var47.x4[0]; + var48.x4[1] = (orc_uint8) var47.x4[1]; + var48.x4[2] = (orc_uint8) var47.x4[2]; + var48.x4[3] = (orc_uint8) var47.x4[3]; + /* 6: mullw */ + var49.x4[0] = (var48.x4[0] * var41.x4[0]) & 0xffff; + var49.x4[1] = (var48.x4[1] * var41.x4[1]) & 0xffff; + var49.x4[2] = (var48.x4[2] * var41.x4[2]) & 0xffff; + var49.x4[3] = (var48.x4[3] * var41.x4[3]) & 0xffff; + /* 7: shruw */ + var50.x4[0] = ((orc_uint16) var49.x4[0]) >> 8; + var50.x4[1] = ((orc_uint16) var49.x4[1]) >> 8; + var50.x4[2] = ((orc_uint16) var49.x4[2]) >> 8; + var50.x4[3] = ((orc_uint16) var49.x4[3]) >> 8; + /* 8: convubw */ + var51.x4[0] = (orc_uint8) var44.x4[0]; + var51.x4[1] = (orc_uint8) var44.x4[1]; + var51.x4[2] = (orc_uint8) var44.x4[2]; + var51.x4[3] = (orc_uint8) var44.x4[3]; + /* 9: mullw */ + var52.x4[0] = (var51.x4[0] * var50.x4[0]) & 0xffff; + var52.x4[1] = (var51.x4[1] * var50.x4[1]) & 0xffff; + var52.x4[2] = (var51.x4[2] * var50.x4[2]) & 0xffff; + var52.x4[3] = (var51.x4[3] * var50.x4[3]) & 0xffff; + /* 11: convubw */ + var54.x4[0] = (orc_uint8) var53.x4[0]; + var54.x4[1] = (orc_uint8) var53.x4[1]; + var54.x4[2] = (orc_uint8) var53.x4[2]; + var54.x4[3] = (orc_uint8) var53.x4[3]; + /* 12: subw */ + var55.x4[0] = var54.x4[0] - var50.x4[0]; + var55.x4[1] = var54.x4[1] - var50.x4[1]; + var55.x4[2] = var54.x4[2] - var50.x4[2]; + var55.x4[3] = var54.x4[3] - var50.x4[3]; + /* 13: loadl */ + var56 = ptr0[i]; + /* 14: convlw */ + var57.i = var56.i; + /* 15: convwb */ + var58 = var57.i; + /* 16: splatbl */ + var59.i = + ((var58 & 0xff) << 24) | ((var58 & 0xff) << 16) | ((var58 & 0xff) << + 8) | (var58 & 0xff); + /* 17: convubw */ + var60.x4[0] = (orc_uint8) var59.x4[0]; + var60.x4[1] = (orc_uint8) var59.x4[1]; + var60.x4[2] = (orc_uint8) var59.x4[2]; + var60.x4[3] = (orc_uint8) var59.x4[3]; + /* 18: mullw */ + var61.x4[0] = (var60.x4[0] * var55.x4[0]) & 0xffff; + var61.x4[1] = (var60.x4[1] * var55.x4[1]) & 0xffff; + var61.x4[2] = (var60.x4[2] * var55.x4[2]) & 0xffff; + var61.x4[3] = (var60.x4[3] * var55.x4[3]) & 0xffff; + /* 19: div255w */ + var62.x4[0] = + ((orc_uint16) (((orc_uint16) (var61.x4[0] + 128)) + + (((orc_uint16) (var61.x4[0] + 128)) >> 8))) >> 8; + var62.x4[1] = + ((orc_uint16) (((orc_uint16) (var61.x4[1] + 128)) + + (((orc_uint16) (var61.x4[1] + 128)) >> 8))) >> 8; + var62.x4[2] = + ((orc_uint16) (((orc_uint16) (var61.x4[2] + 128)) + + (((orc_uint16) (var61.x4[2] + 128)) >> 8))) >> 8; + var62.x4[3] = + ((orc_uint16) (((orc_uint16) (var61.x4[3] + 128)) + + (((orc_uint16) (var61.x4[3] + 128)) >> 8))) >> 8; + /* 20: convubw */ + var63.x4[0] = (orc_uint8) var56.x4[0]; + var63.x4[1] = (orc_uint8) var56.x4[1]; + var63.x4[2] = (orc_uint8) var56.x4[2]; + var63.x4[3] = (orc_uint8) var56.x4[3]; + /* 21: mullw */ + var64.x4[0] = (var63.x4[0] * var62.x4[0]) & 0xffff; + var64.x4[1] = (var63.x4[1] * var62.x4[1]) & 0xffff; + var64.x4[2] = (var63.x4[2] * var62.x4[2]) & 0xffff; + var64.x4[3] = (var63.x4[3] * var62.x4[3]) & 0xffff; + /* 22: addw */ + var65.x4[0] = var64.x4[0] + var52.x4[0]; + var65.x4[1] = var64.x4[1] + var52.x4[1]; + var65.x4[2] = var64.x4[2] + var52.x4[2]; + var65.x4[3] = var64.x4[3] + var52.x4[3]; + /* 23: addw */ + var66.x4[0] = var62.x4[0] + var50.x4[0]; + var66.x4[1] = var62.x4[1] + var50.x4[1]; + var66.x4[2] = var62.x4[2] + var50.x4[2]; + var66.x4[3] = var62.x4[3] + var50.x4[3]; + /* 24: divluw */ + var67.x4[0] = + ((var66.x4[0] & 0xff) == + 0) ? 255 : ORC_CLAMP_UB (((orc_uint16) var65.x4[0]) / + ((orc_uint16) var66.x4[0] & 0xff)); + var67.x4[1] = + ((var66.x4[1] & 0xff) == + 0) ? 255 : ORC_CLAMP_UB (((orc_uint16) var65.x4[1]) / + ((orc_uint16) var66.x4[1] & 0xff)); + var67.x4[2] = + ((var66.x4[2] & 0xff) == + 0) ? 255 : ORC_CLAMP_UB (((orc_uint16) var65.x4[2]) / + ((orc_uint16) var66.x4[2] & 0xff)); + var67.x4[3] = + ((var66.x4[3] & 0xff) == + 0) ? 255 : ORC_CLAMP_UB (((orc_uint16) var65.x4[3]) / + ((orc_uint16) var66.x4[3] & 0xff)); + /* 25: convwb */ + var68.x4[0] = var67.x4[0]; + var68.x4[1] = var67.x4[1]; + var68.x4[2] = var67.x4[2]; + var68.x4[3] = var67.x4[3]; + /* 27: andl */ + var69.i = var68.i & var42.i; + /* 28: convwb */ + var70.x4[0] = var66.x4[0]; + var70.x4[1] = var66.x4[1]; + var70.x4[2] = var66.x4[2]; + var70.x4[3] = var66.x4[3]; + /* 30: andl */ + var71.i = var70.i & var43.i; + /* 31: orl */ + var72.i = var69.i | var71.i; + /* 32: storel */ + ptr0[i] = var72; + } + } + +} + +#else +static void +_backup_compositor_orc_overlay_argb (OrcExecutor * ORC_RESTRICT ex) +{ + int i; + int j; + int n = ex->n; + int m = ex->params[ORC_VAR_A1]; + orc_union32 *ORC_RESTRICT ptr0; + const orc_union32 *ORC_RESTRICT ptr4; + orc_union64 var41; +#if defined(__APPLE__) && __GNUC__ == 4 && __GNUC_MINOR__ == 2 && defined (__i386__) + volatile orc_union32 var42; +#else + orc_union32 var42; +#endif +#if defined(__APPLE__) && __GNUC__ == 4 && __GNUC_MINOR__ == 2 && defined (__i386__) + volatile orc_union32 var43; +#else + orc_union32 var43; +#endif + orc_union32 var44; + orc_union16 var45; + orc_int8 var46; + orc_union32 var47; + orc_union64 var48; + orc_union64 var49; + orc_union64 var50; + orc_union64 var51; + orc_union64 var52; + orc_union32 var53; + orc_union64 var54; + orc_union64 var55; + orc_union32 var56; + orc_union16 var57; + orc_int8 var58; + orc_union32 var59; + orc_union64 var60; + orc_union64 var61; + orc_union64 var62; + orc_union64 var63; + orc_union64 var64; + orc_union64 var65; + orc_union64 var66; + orc_union64 var67; + orc_union32 var68; + orc_union32 var69; + orc_union32 var70; + orc_union32 var71; + orc_union32 var72; + + for (j = 0; j < m; j++) { + ptr0 = ORC_PTR_OFFSET (ex->arrays[0], ex->params[0] * j); + ptr4 = ORC_PTR_OFFSET (ex->arrays[4], ex->params[4] * j); + + /* 5: loadpw */ + var41.x4[0] = ex->params[24]; + var41.x4[1] = ex->params[24]; + var41.x4[2] = ex->params[24]; + var41.x4[3] = ex->params[24]; + /* 10: loadpl */ + var53.i = (int) 0xffffffff; /* -1 or 2.122e-314f */ + /* 26: loadpl */ + var42.i = (int) 0xffffff00; /* -256 or 2.122e-314f */ + /* 29: loadpl */ + var43.i = (int) 0x000000ff; /* 255 or 1.25987e-321f */ + + for (i = 0; i < n; i++) { + /* 0: loadl */ + var44 = ptr4[i]; + /* 1: convlw */ + var45.i = var44.i; + /* 2: convwb */ + var46 = var45.i; + /* 3: splatbl */ + var47.i = + ((var46 & 0xff) << 24) | ((var46 & 0xff) << 16) | ((var46 & 0xff) << + 8) | (var46 & 0xff); + /* 4: convubw */ + var48.x4[0] = (orc_uint8) var47.x4[0]; + var48.x4[1] = (orc_uint8) var47.x4[1]; + var48.x4[2] = (orc_uint8) var47.x4[2]; + var48.x4[3] = (orc_uint8) var47.x4[3]; + /* 6: mullw */ + var49.x4[0] = (var48.x4[0] * var41.x4[0]) & 0xffff; + var49.x4[1] = (var48.x4[1] * var41.x4[1]) & 0xffff; + var49.x4[2] = (var48.x4[2] * var41.x4[2]) & 0xffff; + var49.x4[3] = (var48.x4[3] * var41.x4[3]) & 0xffff; + /* 7: shruw */ + var50.x4[0] = ((orc_uint16) var49.x4[0]) >> 8; + var50.x4[1] = ((orc_uint16) var49.x4[1]) >> 8; + var50.x4[2] = ((orc_uint16) var49.x4[2]) >> 8; + var50.x4[3] = ((orc_uint16) var49.x4[3]) >> 8; + /* 8: convubw */ + var51.x4[0] = (orc_uint8) var44.x4[0]; + var51.x4[1] = (orc_uint8) var44.x4[1]; + var51.x4[2] = (orc_uint8) var44.x4[2]; + var51.x4[3] = (orc_uint8) var44.x4[3]; + /* 9: mullw */ + var52.x4[0] = (var51.x4[0] * var50.x4[0]) & 0xffff; + var52.x4[1] = (var51.x4[1] * var50.x4[1]) & 0xffff; + var52.x4[2] = (var51.x4[2] * var50.x4[2]) & 0xffff; + var52.x4[3] = (var51.x4[3] * var50.x4[3]) & 0xffff; + /* 11: convubw */ + var54.x4[0] = (orc_uint8) var53.x4[0]; + var54.x4[1] = (orc_uint8) var53.x4[1]; + var54.x4[2] = (orc_uint8) var53.x4[2]; + var54.x4[3] = (orc_uint8) var53.x4[3]; + /* 12: subw */ + var55.x4[0] = var54.x4[0] - var50.x4[0]; + var55.x4[1] = var54.x4[1] - var50.x4[1]; + var55.x4[2] = var54.x4[2] - var50.x4[2]; + var55.x4[3] = var54.x4[3] - var50.x4[3]; + /* 13: loadl */ + var56 = ptr0[i]; + /* 14: convlw */ + var57.i = var56.i; + /* 15: convwb */ + var58 = var57.i; + /* 16: splatbl */ + var59.i = + ((var58 & 0xff) << 24) | ((var58 & 0xff) << 16) | ((var58 & 0xff) << + 8) | (var58 & 0xff); + /* 17: convubw */ + var60.x4[0] = (orc_uint8) var59.x4[0]; + var60.x4[1] = (orc_uint8) var59.x4[1]; + var60.x4[2] = (orc_uint8) var59.x4[2]; + var60.x4[3] = (orc_uint8) var59.x4[3]; + /* 18: mullw */ + var61.x4[0] = (var60.x4[0] * var55.x4[0]) & 0xffff; + var61.x4[1] = (var60.x4[1] * var55.x4[1]) & 0xffff; + var61.x4[2] = (var60.x4[2] * var55.x4[2]) & 0xffff; + var61.x4[3] = (var60.x4[3] * var55.x4[3]) & 0xffff; + /* 19: div255w */ + var62.x4[0] = + ((orc_uint16) (((orc_uint16) (var61.x4[0] + 128)) + + (((orc_uint16) (var61.x4[0] + 128)) >> 8))) >> 8; + var62.x4[1] = + ((orc_uint16) (((orc_uint16) (var61.x4[1] + 128)) + + (((orc_uint16) (var61.x4[1] + 128)) >> 8))) >> 8; + var62.x4[2] = + ((orc_uint16) (((orc_uint16) (var61.x4[2] + 128)) + + (((orc_uint16) (var61.x4[2] + 128)) >> 8))) >> 8; + var62.x4[3] = + ((orc_uint16) (((orc_uint16) (var61.x4[3] + 128)) + + (((orc_uint16) (var61.x4[3] + 128)) >> 8))) >> 8; + /* 20: convubw */ + var63.x4[0] = (orc_uint8) var56.x4[0]; + var63.x4[1] = (orc_uint8) var56.x4[1]; + var63.x4[2] = (orc_uint8) var56.x4[2]; + var63.x4[3] = (orc_uint8) var56.x4[3]; + /* 21: mullw */ + var64.x4[0] = (var63.x4[0] * var62.x4[0]) & 0xffff; + var64.x4[1] = (var63.x4[1] * var62.x4[1]) & 0xffff; + var64.x4[2] = (var63.x4[2] * var62.x4[2]) & 0xffff; + var64.x4[3] = (var63.x4[3] * var62.x4[3]) & 0xffff; + /* 22: addw */ + var65.x4[0] = var64.x4[0] + var52.x4[0]; + var65.x4[1] = var64.x4[1] + var52.x4[1]; + var65.x4[2] = var64.x4[2] + var52.x4[2]; + var65.x4[3] = var64.x4[3] + var52.x4[3]; + /* 23: addw */ + var66.x4[0] = var62.x4[0] + var50.x4[0]; + var66.x4[1] = var62.x4[1] + var50.x4[1]; + var66.x4[2] = var62.x4[2] + var50.x4[2]; + var66.x4[3] = var62.x4[3] + var50.x4[3]; + /* 24: divluw */ + var67.x4[0] = + ((var66.x4[0] & 0xff) == + 0) ? 255 : ORC_CLAMP_UB (((orc_uint16) var65.x4[0]) / + ((orc_uint16) var66.x4[0] & 0xff)); + var67.x4[1] = + ((var66.x4[1] & 0xff) == + 0) ? 255 : ORC_CLAMP_UB (((orc_uint16) var65.x4[1]) / + ((orc_uint16) var66.x4[1] & 0xff)); + var67.x4[2] = + ((var66.x4[2] & 0xff) == + 0) ? 255 : ORC_CLAMP_UB (((orc_uint16) var65.x4[2]) / + ((orc_uint16) var66.x4[2] & 0xff)); + var67.x4[3] = + ((var66.x4[3] & 0xff) == + 0) ? 255 : ORC_CLAMP_UB (((orc_uint16) var65.x4[3]) / + ((orc_uint16) var66.x4[3] & 0xff)); + /* 25: convwb */ + var68.x4[0] = var67.x4[0]; + var68.x4[1] = var67.x4[1]; + var68.x4[2] = var67.x4[2]; + var68.x4[3] = var67.x4[3]; + /* 27: andl */ + var69.i = var68.i & var42.i; + /* 28: convwb */ + var70.x4[0] = var66.x4[0]; + var70.x4[1] = var66.x4[1]; + var70.x4[2] = var66.x4[2]; + var70.x4[3] = var66.x4[3]; + /* 30: andl */ + var71.i = var70.i & var43.i; + /* 31: orl */ + var72.i = var69.i | var71.i; + /* 32: storel */ + ptr0[i] = var72; + } + } + +} + +void +compositor_orc_overlay_argb (guint8 * ORC_RESTRICT d1, int d1_stride, + const guint8 * ORC_RESTRICT s1, int s1_stride, int p1, int n, int m) +{ + OrcExecutor _ex, *ex = &_ex; + static volatile int p_inited = 0; + static OrcCode *c = 0; + void (*func) (OrcExecutor *); + + if (!p_inited) { + orc_once_mutex_lock (); + if (!p_inited) { + OrcProgram *p; + +#if 1 + static const orc_uint8 bc[] = { + 1, 7, 9, 27, 99, 111, 109, 112, 111, 115, 105, 116, 111, 114, 95, 111, + 114, 99, 95, 111, 118, 101, 114, 108, 97, 121, 95, 97, 114, 103, 98, 11, + 4, 4, 12, 4, 4, 14, 4, 255, 255, 255, 255, 14, 4, 255, 0, 0, + 0, 14, 4, 0, 255, 255, 255, 14, 4, 8, 0, 0, 0, 16, 2, 20, + 4, 20, 2, 20, 1, 20, 8, 20, 8, 20, 8, 20, 4, 20, 8, 20, + 8, 113, 32, 4, 163, 33, 32, 157, 34, 33, 152, 38, 34, 21, 2, 150, + 35, 38, 21, 2, 89, 35, 35, 24, 21, 2, 95, 35, 35, 19, 21, 2, + 150, 40, 32, 21, 2, 89, 40, 40, 35, 115, 38, 16, 21, 2, 150, 36, + 38, 21, 2, 98, 36, 36, 35, 113, 32, 0, 163, 33, 32, 157, 34, 33, + 152, 38, 34, 21, 2, 150, 37, 38, 21, 2, 89, 37, 37, 36, 21, 2, + 80, 37, 37, 21, 2, 150, 39, 32, 21, 2, 89, 39, 39, 37, 21, 2, + 70, 39, 39, 40, 21, 2, 70, 37, 37, 35, 21, 2, 81, 39, 39, 37, + 21, 2, 157, 32, 39, 106, 32, 32, 18, 21, 2, 157, 38, 37, 106, 38, + 38, 17, 123, 32, 32, 38, 128, 0, 32, 2, 0, + }; + p = orc_program_new_from_static_bytecode (bc); + orc_program_set_backup_function (p, _backup_compositor_orc_overlay_argb); +#else + p = orc_program_new (); + orc_program_set_2d (p); + orc_program_set_name (p, "compositor_orc_overlay_argb"); + orc_program_set_backup_function (p, _backup_compositor_orc_overlay_argb); + orc_program_add_destination (p, 4, "d1"); + orc_program_add_source (p, 4, "s1"); + orc_program_add_constant (p, 4, 0xffffffff, "c1"); + orc_program_add_constant (p, 4, 0x000000ff, "c2"); + orc_program_add_constant (p, 4, 0xffffff00, "c3"); + orc_program_add_constant (p, 4, 0x00000008, "c4"); + orc_program_add_parameter (p, 2, "p1"); + orc_program_add_temporary (p, 4, "t1"); + orc_program_add_temporary (p, 2, "t2"); + orc_program_add_temporary (p, 1, "t3"); + orc_program_add_temporary (p, 8, "t4"); + orc_program_add_temporary (p, 8, "t5"); + orc_program_add_temporary (p, 8, "t6"); + orc_program_add_temporary (p, 4, "t7"); + orc_program_add_temporary (p, 8, "t8"); + orc_program_add_temporary (p, 8, "t9"); + + orc_program_append_2 (p, "loadl", 0, ORC_VAR_T1, ORC_VAR_S1, ORC_VAR_D1, + ORC_VAR_D1); + orc_program_append_2 (p, "convlw", 0, ORC_VAR_T2, ORC_VAR_T1, ORC_VAR_D1, + ORC_VAR_D1); + orc_program_append_2 (p, "convwb", 0, ORC_VAR_T3, ORC_VAR_T2, ORC_VAR_D1, + ORC_VAR_D1); + orc_program_append_2 (p, "splatbl", 0, ORC_VAR_T7, ORC_VAR_T3, ORC_VAR_D1, + ORC_VAR_D1); + orc_program_append_2 (p, "convubw", 2, ORC_VAR_T4, ORC_VAR_T7, ORC_VAR_D1, + ORC_VAR_D1); + orc_program_append_2 (p, "mullw", 2, ORC_VAR_T4, ORC_VAR_T4, ORC_VAR_P1, + ORC_VAR_D1); + orc_program_append_2 (p, "shruw", 2, ORC_VAR_T4, ORC_VAR_T4, ORC_VAR_C4, + ORC_VAR_D1); + orc_program_append_2 (p, "convubw", 2, ORC_VAR_T9, ORC_VAR_T1, ORC_VAR_D1, + ORC_VAR_D1); + orc_program_append_2 (p, "mullw", 2, ORC_VAR_T9, ORC_VAR_T9, ORC_VAR_T4, + ORC_VAR_D1); + orc_program_append_2 (p, "loadpl", 0, ORC_VAR_T7, ORC_VAR_C1, ORC_VAR_D1, + ORC_VAR_D1); + orc_program_append_2 (p, "convubw", 2, ORC_VAR_T5, ORC_VAR_T7, ORC_VAR_D1, + ORC_VAR_D1); + orc_program_append_2 (p, "subw", 2, ORC_VAR_T5, ORC_VAR_T5, ORC_VAR_T4, + ORC_VAR_D1); + orc_program_append_2 (p, "loadl", 0, ORC_VAR_T1, ORC_VAR_D1, ORC_VAR_D1, + ORC_VAR_D1); + orc_program_append_2 (p, "convlw", 0, ORC_VAR_T2, ORC_VAR_T1, ORC_VAR_D1, + ORC_VAR_D1); + orc_program_append_2 (p, "convwb", 0, ORC_VAR_T3, ORC_VAR_T2, ORC_VAR_D1, + ORC_VAR_D1); + orc_program_append_2 (p, "splatbl", 0, ORC_VAR_T7, ORC_VAR_T3, ORC_VAR_D1, + ORC_VAR_D1); + orc_program_append_2 (p, "convubw", 2, ORC_VAR_T6, ORC_VAR_T7, ORC_VAR_D1, + ORC_VAR_D1); + orc_program_append_2 (p, "mullw", 2, ORC_VAR_T6, ORC_VAR_T6, ORC_VAR_T5, + ORC_VAR_D1); + orc_program_append_2 (p, "div255w", 2, ORC_VAR_T6, ORC_VAR_T6, ORC_VAR_D1, + ORC_VAR_D1); + orc_program_append_2 (p, "convubw", 2, ORC_VAR_T8, ORC_VAR_T1, ORC_VAR_D1, + ORC_VAR_D1); + orc_program_append_2 (p, "mullw", 2, ORC_VAR_T8, ORC_VAR_T8, ORC_VAR_T6, + ORC_VAR_D1); + orc_program_append_2 (p, "addw", 2, ORC_VAR_T8, ORC_VAR_T8, ORC_VAR_T9, + ORC_VAR_D1); + orc_program_append_2 (p, "addw", 2, ORC_VAR_T6, ORC_VAR_T6, ORC_VAR_T4, + ORC_VAR_D1); + orc_program_append_2 (p, "divluw", 2, ORC_VAR_T8, ORC_VAR_T8, ORC_VAR_T6, + ORC_VAR_D1); + orc_program_append_2 (p, "convwb", 2, ORC_VAR_T1, ORC_VAR_T8, ORC_VAR_D1, + ORC_VAR_D1); + orc_program_append_2 (p, "andl", 0, ORC_VAR_T1, ORC_VAR_T1, ORC_VAR_C3, + ORC_VAR_D1); + orc_program_append_2 (p, "convwb", 2, ORC_VAR_T7, ORC_VAR_T6, ORC_VAR_D1, + ORC_VAR_D1); + orc_program_append_2 (p, "andl", 0, ORC_VAR_T7, ORC_VAR_T7, ORC_VAR_C2, + ORC_VAR_D1); + orc_program_append_2 (p, "orl", 0, ORC_VAR_T1, ORC_VAR_T1, ORC_VAR_T7, + ORC_VAR_D1); + orc_program_append_2 (p, "storel", 0, ORC_VAR_D1, ORC_VAR_T1, ORC_VAR_D1, + ORC_VAR_D1); +#endif + + orc_program_compile (p); + c = orc_program_take_code (p); + orc_program_free (p); + } + p_inited = TRUE; + orc_once_mutex_unlock (); + } + ex->arrays[ORC_VAR_A2] = c; + ex->program = 0; + + ex->n = n; + ORC_EXECUTOR_M (ex) = m; + ex->arrays[ORC_VAR_D1] = d1; + ex->params[ORC_VAR_D1] = d1_stride; + ex->arrays[ORC_VAR_S1] = (void *) s1; + ex->params[ORC_VAR_S1] = s1_stride; + ex->params[ORC_VAR_P1] = p1; + + func = c->exec; + func (ex); +} +#endif + + +/* compositor_orc_overlay_bgra */ +#ifdef DISABLE_ORC +void +compositor_orc_overlay_bgra (guint8 * ORC_RESTRICT d1, int d1_stride, + const guint8 * ORC_RESTRICT s1, int s1_stride, int p1, int n, int m) +{ + int i; + int j; + orc_union32 *ORC_RESTRICT ptr0; + const orc_union32 *ORC_RESTRICT ptr4; + orc_union64 var42; +#if defined(__APPLE__) && __GNUC__ == 4 && __GNUC_MINOR__ == 2 && defined (__i386__) + volatile orc_union32 var43; +#else + orc_union32 var43; +#endif +#if defined(__APPLE__) && __GNUC__ == 4 && __GNUC_MINOR__ == 2 && defined (__i386__) + volatile orc_union32 var44; +#else + orc_union32 var44; +#endif + orc_union32 var45; + orc_union32 var46; + orc_union16 var47; + orc_int8 var48; + orc_union32 var49; + orc_union64 var50; + orc_union64 var51; + orc_union64 var52; + orc_union64 var53; + orc_union64 var54; + orc_union32 var55; + orc_union64 var56; + orc_union64 var57; + orc_union32 var58; + orc_union32 var59; + orc_union16 var60; + orc_int8 var61; + orc_union32 var62; + orc_union64 var63; + orc_union64 var64; + orc_union64 var65; + orc_union64 var66; + orc_union64 var67; + orc_union64 var68; + orc_union64 var69; + orc_union64 var70; + orc_union32 var71; + orc_union32 var72; + orc_union32 var73; + orc_union32 var74; + orc_union32 var75; + + for (j = 0; j < m; j++) { + ptr0 = ORC_PTR_OFFSET (d1, d1_stride * j); + ptr4 = ORC_PTR_OFFSET (s1, s1_stride * j); + + /* 6: loadpw */ + var42.x4[0] = p1; + var42.x4[1] = p1; + var42.x4[2] = p1; + var42.x4[3] = p1; + /* 11: loadpl */ + var55.i = (int) 0xffffffff; /* -1 or 2.122e-314f */ + /* 28: loadpl */ + var43.i = (int) 0x00ffffff; /* 16777215 or 8.28905e-317f */ + /* 31: loadpl */ + var44.i = (int) 0xff000000; /* -16777216 or 2.11371e-314f */ + + for (i = 0; i < n; i++) { + /* 0: loadl */ + var45 = ptr4[i]; + /* 1: shrul */ + var46.i = ((orc_uint32) var45.i) >> 24; + /* 2: convlw */ + var47.i = var46.i; + /* 3: convwb */ + var48 = var47.i; + /* 4: splatbl */ + var49.i = + ((var48 & 0xff) << 24) | ((var48 & 0xff) << 16) | ((var48 & 0xff) << + 8) | (var48 & 0xff); + /* 5: convubw */ + var50.x4[0] = (orc_uint8) var49.x4[0]; + var50.x4[1] = (orc_uint8) var49.x4[1]; + var50.x4[2] = (orc_uint8) var49.x4[2]; + var50.x4[3] = (orc_uint8) var49.x4[3]; + /* 7: mullw */ + var51.x4[0] = (var50.x4[0] * var42.x4[0]) & 0xffff; + var51.x4[1] = (var50.x4[1] * var42.x4[1]) & 0xffff; + var51.x4[2] = (var50.x4[2] * var42.x4[2]) & 0xffff; + var51.x4[3] = (var50.x4[3] * var42.x4[3]) & 0xffff; + /* 8: shruw */ + var52.x4[0] = ((orc_uint16) var51.x4[0]) >> 8; + var52.x4[1] = ((orc_uint16) var51.x4[1]) >> 8; + var52.x4[2] = ((orc_uint16) var51.x4[2]) >> 8; + var52.x4[3] = ((orc_uint16) var51.x4[3]) >> 8; + /* 9: convubw */ + var53.x4[0] = (orc_uint8) var45.x4[0]; + var53.x4[1] = (orc_uint8) var45.x4[1]; + var53.x4[2] = (orc_uint8) var45.x4[2]; + var53.x4[3] = (orc_uint8) var45.x4[3]; + /* 10: mullw */ + var54.x4[0] = (var53.x4[0] * var52.x4[0]) & 0xffff; + var54.x4[1] = (var53.x4[1] * var52.x4[1]) & 0xffff; + var54.x4[2] = (var53.x4[2] * var52.x4[2]) & 0xffff; + var54.x4[3] = (var53.x4[3] * var52.x4[3]) & 0xffff; + /* 12: convubw */ + var56.x4[0] = (orc_uint8) var55.x4[0]; + var56.x4[1] = (orc_uint8) var55.x4[1]; + var56.x4[2] = (orc_uint8) var55.x4[2]; + var56.x4[3] = (orc_uint8) var55.x4[3]; + /* 13: subw */ + var57.x4[0] = var56.x4[0] - var52.x4[0]; + var57.x4[1] = var56.x4[1] - var52.x4[1]; + var57.x4[2] = var56.x4[2] - var52.x4[2]; + var57.x4[3] = var56.x4[3] - var52.x4[3]; + /* 14: loadl */ + var58 = ptr0[i]; + /* 15: shrul */ + var59.i = ((orc_uint32) var58.i) >> 24; + /* 16: convlw */ + var60.i = var59.i; + /* 17: convwb */ + var61 = var60.i; + /* 18: splatbl */ + var62.i = + ((var61 & 0xff) << 24) | ((var61 & 0xff) << 16) | ((var61 & 0xff) << + 8) | (var61 & 0xff); + /* 19: convubw */ + var63.x4[0] = (orc_uint8) var62.x4[0]; + var63.x4[1] = (orc_uint8) var62.x4[1]; + var63.x4[2] = (orc_uint8) var62.x4[2]; + var63.x4[3] = (orc_uint8) var62.x4[3]; + /* 20: mullw */ + var64.x4[0] = (var63.x4[0] * var57.x4[0]) & 0xffff; + var64.x4[1] = (var63.x4[1] * var57.x4[1]) & 0xffff; + var64.x4[2] = (var63.x4[2] * var57.x4[2]) & 0xffff; + var64.x4[3] = (var63.x4[3] * var57.x4[3]) & 0xffff; + /* 21: div255w */ + var65.x4[0] = + ((orc_uint16) (((orc_uint16) (var64.x4[0] + 128)) + + (((orc_uint16) (var64.x4[0] + 128)) >> 8))) >> 8; + var65.x4[1] = + ((orc_uint16) (((orc_uint16) (var64.x4[1] + 128)) + + (((orc_uint16) (var64.x4[1] + 128)) >> 8))) >> 8; + var65.x4[2] = + ((orc_uint16) (((orc_uint16) (var64.x4[2] + 128)) + + (((orc_uint16) (var64.x4[2] + 128)) >> 8))) >> 8; + var65.x4[3] = + ((orc_uint16) (((orc_uint16) (var64.x4[3] + 128)) + + (((orc_uint16) (var64.x4[3] + 128)) >> 8))) >> 8; + /* 22: convubw */ + var66.x4[0] = (orc_uint8) var58.x4[0]; + var66.x4[1] = (orc_uint8) var58.x4[1]; + var66.x4[2] = (orc_uint8) var58.x4[2]; + var66.x4[3] = (orc_uint8) var58.x4[3]; + /* 23: mullw */ + var67.x4[0] = (var66.x4[0] * var65.x4[0]) & 0xffff; + var67.x4[1] = (var66.x4[1] * var65.x4[1]) & 0xffff; + var67.x4[2] = (var66.x4[2] * var65.x4[2]) & 0xffff; + var67.x4[3] = (var66.x4[3] * var65.x4[3]) & 0xffff; + /* 24: addw */ + var68.x4[0] = var67.x4[0] + var54.x4[0]; + var68.x4[1] = var67.x4[1] + var54.x4[1]; + var68.x4[2] = var67.x4[2] + var54.x4[2]; + var68.x4[3] = var67.x4[3] + var54.x4[3]; + /* 25: addw */ + var69.x4[0] = var65.x4[0] + var52.x4[0]; + var69.x4[1] = var65.x4[1] + var52.x4[1]; + var69.x4[2] = var65.x4[2] + var52.x4[2]; + var69.x4[3] = var65.x4[3] + var52.x4[3]; + /* 26: divluw */ + var70.x4[0] = + ((var69.x4[0] & 0xff) == + 0) ? 255 : ORC_CLAMP_UB (((orc_uint16) var68.x4[0]) / + ((orc_uint16) var69.x4[0] & 0xff)); + var70.x4[1] = + ((var69.x4[1] & 0xff) == + 0) ? 255 : ORC_CLAMP_UB (((orc_uint16) var68.x4[1]) / + ((orc_uint16) var69.x4[1] & 0xff)); + var70.x4[2] = + ((var69.x4[2] & 0xff) == + 0) ? 255 : ORC_CLAMP_UB (((orc_uint16) var68.x4[2]) / + ((orc_uint16) var69.x4[2] & 0xff)); + var70.x4[3] = + ((var69.x4[3] & 0xff) == + 0) ? 255 : ORC_CLAMP_UB (((orc_uint16) var68.x4[3]) / + ((orc_uint16) var69.x4[3] & 0xff)); + /* 27: convwb */ + var71.x4[0] = var70.x4[0]; + var71.x4[1] = var70.x4[1]; + var71.x4[2] = var70.x4[2]; + var71.x4[3] = var70.x4[3]; + /* 29: andl */ + var72.i = var71.i & var43.i; + /* 30: convwb */ + var73.x4[0] = var69.x4[0]; + var73.x4[1] = var69.x4[1]; + var73.x4[2] = var69.x4[2]; + var73.x4[3] = var69.x4[3]; + /* 32: andl */ + var74.i = var73.i & var44.i; + /* 33: orl */ + var75.i = var72.i | var74.i; + /* 34: storel */ + ptr0[i] = var75; + } + } + +} + +#else +static void +_backup_compositor_orc_overlay_bgra (OrcExecutor * ORC_RESTRICT ex) +{ + int i; + int j; + int n = ex->n; + int m = ex->params[ORC_VAR_A1]; + orc_union32 *ORC_RESTRICT ptr0; + const orc_union32 *ORC_RESTRICT ptr4; + orc_union64 var42; +#if defined(__APPLE__) && __GNUC__ == 4 && __GNUC_MINOR__ == 2 && defined (__i386__) + volatile orc_union32 var43; +#else + orc_union32 var43; +#endif +#if defined(__APPLE__) && __GNUC__ == 4 && __GNUC_MINOR__ == 2 && defined (__i386__) + volatile orc_union32 var44; +#else + orc_union32 var44; +#endif + orc_union32 var45; + orc_union32 var46; + orc_union16 var47; + orc_int8 var48; + orc_union32 var49; + orc_union64 var50; + orc_union64 var51; + orc_union64 var52; + orc_union64 var53; + orc_union64 var54; + orc_union32 var55; + orc_union64 var56; + orc_union64 var57; + orc_union32 var58; + orc_union32 var59; + orc_union16 var60; + orc_int8 var61; + orc_union32 var62; + orc_union64 var63; + orc_union64 var64; + orc_union64 var65; + orc_union64 var66; + orc_union64 var67; + orc_union64 var68; + orc_union64 var69; + orc_union64 var70; + orc_union32 var71; + orc_union32 var72; + orc_union32 var73; + orc_union32 var74; + orc_union32 var75; + + for (j = 0; j < m; j++) { + ptr0 = ORC_PTR_OFFSET (ex->arrays[0], ex->params[0] * j); + ptr4 = ORC_PTR_OFFSET (ex->arrays[4], ex->params[4] * j); + + /* 6: loadpw */ + var42.x4[0] = ex->params[24]; + var42.x4[1] = ex->params[24]; + var42.x4[2] = ex->params[24]; + var42.x4[3] = ex->params[24]; + /* 11: loadpl */ + var55.i = (int) 0xffffffff; /* -1 or 2.122e-314f */ + /* 28: loadpl */ + var43.i = (int) 0x00ffffff; /* 16777215 or 8.28905e-317f */ + /* 31: loadpl */ + var44.i = (int) 0xff000000; /* -16777216 or 2.11371e-314f */ + + for (i = 0; i < n; i++) { + /* 0: loadl */ + var45 = ptr4[i]; + /* 1: shrul */ + var46.i = ((orc_uint32) var45.i) >> 24; + /* 2: convlw */ + var47.i = var46.i; + /* 3: convwb */ + var48 = var47.i; + /* 4: splatbl */ + var49.i = + ((var48 & 0xff) << 24) | ((var48 & 0xff) << 16) | ((var48 & 0xff) << + 8) | (var48 & 0xff); + /* 5: convubw */ + var50.x4[0] = (orc_uint8) var49.x4[0]; + var50.x4[1] = (orc_uint8) var49.x4[1]; + var50.x4[2] = (orc_uint8) var49.x4[2]; + var50.x4[3] = (orc_uint8) var49.x4[3]; + /* 7: mullw */ + var51.x4[0] = (var50.x4[0] * var42.x4[0]) & 0xffff; + var51.x4[1] = (var50.x4[1] * var42.x4[1]) & 0xffff; + var51.x4[2] = (var50.x4[2] * var42.x4[2]) & 0xffff; + var51.x4[3] = (var50.x4[3] * var42.x4[3]) & 0xffff; + /* 8: shruw */ + var52.x4[0] = ((orc_uint16) var51.x4[0]) >> 8; + var52.x4[1] = ((orc_uint16) var51.x4[1]) >> 8; + var52.x4[2] = ((orc_uint16) var51.x4[2]) >> 8; + var52.x4[3] = ((orc_uint16) var51.x4[3]) >> 8; + /* 9: convubw */ + var53.x4[0] = (orc_uint8) var45.x4[0]; + var53.x4[1] = (orc_uint8) var45.x4[1]; + var53.x4[2] = (orc_uint8) var45.x4[2]; + var53.x4[3] = (orc_uint8) var45.x4[3]; + /* 10: mullw */ + var54.x4[0] = (var53.x4[0] * var52.x4[0]) & 0xffff; + var54.x4[1] = (var53.x4[1] * var52.x4[1]) & 0xffff; + var54.x4[2] = (var53.x4[2] * var52.x4[2]) & 0xffff; + var54.x4[3] = (var53.x4[3] * var52.x4[3]) & 0xffff; + /* 12: convubw */ + var56.x4[0] = (orc_uint8) var55.x4[0]; + var56.x4[1] = (orc_uint8) var55.x4[1]; + var56.x4[2] = (orc_uint8) var55.x4[2]; + var56.x4[3] = (orc_uint8) var55.x4[3]; + /* 13: subw */ + var57.x4[0] = var56.x4[0] - var52.x4[0]; + var57.x4[1] = var56.x4[1] - var52.x4[1]; + var57.x4[2] = var56.x4[2] - var52.x4[2]; + var57.x4[3] = var56.x4[3] - var52.x4[3]; + /* 14: loadl */ + var58 = ptr0[i]; + /* 15: shrul */ + var59.i = ((orc_uint32) var58.i) >> 24; + /* 16: convlw */ + var60.i = var59.i; + /* 17: convwb */ + var61 = var60.i; + /* 18: splatbl */ + var62.i = + ((var61 & 0xff) << 24) | ((var61 & 0xff) << 16) | ((var61 & 0xff) << + 8) | (var61 & 0xff); + /* 19: convubw */ + var63.x4[0] = (orc_uint8) var62.x4[0]; + var63.x4[1] = (orc_uint8) var62.x4[1]; + var63.x4[2] = (orc_uint8) var62.x4[2]; + var63.x4[3] = (orc_uint8) var62.x4[3]; + /* 20: mullw */ + var64.x4[0] = (var63.x4[0] * var57.x4[0]) & 0xffff; + var64.x4[1] = (var63.x4[1] * var57.x4[1]) & 0xffff; + var64.x4[2] = (var63.x4[2] * var57.x4[2]) & 0xffff; + var64.x4[3] = (var63.x4[3] * var57.x4[3]) & 0xffff; + /* 21: div255w */ + var65.x4[0] = + ((orc_uint16) (((orc_uint16) (var64.x4[0] + 128)) + + (((orc_uint16) (var64.x4[0] + 128)) >> 8))) >> 8; + var65.x4[1] = + ((orc_uint16) (((orc_uint16) (var64.x4[1] + 128)) + + (((orc_uint16) (var64.x4[1] + 128)) >> 8))) >> 8; + var65.x4[2] = + ((orc_uint16) (((orc_uint16) (var64.x4[2] + 128)) + + (((orc_uint16) (var64.x4[2] + 128)) >> 8))) >> 8; + var65.x4[3] = + ((orc_uint16) (((orc_uint16) (var64.x4[3] + 128)) + + (((orc_uint16) (var64.x4[3] + 128)) >> 8))) >> 8; + /* 22: convubw */ + var66.x4[0] = (orc_uint8) var58.x4[0]; + var66.x4[1] = (orc_uint8) var58.x4[1]; + var66.x4[2] = (orc_uint8) var58.x4[2]; + var66.x4[3] = (orc_uint8) var58.x4[3]; + /* 23: mullw */ + var67.x4[0] = (var66.x4[0] * var65.x4[0]) & 0xffff; + var67.x4[1] = (var66.x4[1] * var65.x4[1]) & 0xffff; + var67.x4[2] = (var66.x4[2] * var65.x4[2]) & 0xffff; + var67.x4[3] = (var66.x4[3] * var65.x4[3]) & 0xffff; + /* 24: addw */ + var68.x4[0] = var67.x4[0] + var54.x4[0]; + var68.x4[1] = var67.x4[1] + var54.x4[1]; + var68.x4[2] = var67.x4[2] + var54.x4[2]; + var68.x4[3] = var67.x4[3] + var54.x4[3]; + /* 25: addw */ + var69.x4[0] = var65.x4[0] + var52.x4[0]; + var69.x4[1] = var65.x4[1] + var52.x4[1]; + var69.x4[2] = var65.x4[2] + var52.x4[2]; + var69.x4[3] = var65.x4[3] + var52.x4[3]; + /* 26: divluw */ + var70.x4[0] = + ((var69.x4[0] & 0xff) == + 0) ? 255 : ORC_CLAMP_UB (((orc_uint16) var68.x4[0]) / + ((orc_uint16) var69.x4[0] & 0xff)); + var70.x4[1] = + ((var69.x4[1] & 0xff) == + 0) ? 255 : ORC_CLAMP_UB (((orc_uint16) var68.x4[1]) / + ((orc_uint16) var69.x4[1] & 0xff)); + var70.x4[2] = + ((var69.x4[2] & 0xff) == + 0) ? 255 : ORC_CLAMP_UB (((orc_uint16) var68.x4[2]) / + ((orc_uint16) var69.x4[2] & 0xff)); + var70.x4[3] = + ((var69.x4[3] & 0xff) == + 0) ? 255 : ORC_CLAMP_UB (((orc_uint16) var68.x4[3]) / + ((orc_uint16) var69.x4[3] & 0xff)); + /* 27: convwb */ + var71.x4[0] = var70.x4[0]; + var71.x4[1] = var70.x4[1]; + var71.x4[2] = var70.x4[2]; + var71.x4[3] = var70.x4[3]; + /* 29: andl */ + var72.i = var71.i & var43.i; + /* 30: convwb */ + var73.x4[0] = var69.x4[0]; + var73.x4[1] = var69.x4[1]; + var73.x4[2] = var69.x4[2]; + var73.x4[3] = var69.x4[3]; + /* 32: andl */ + var74.i = var73.i & var44.i; + /* 33: orl */ + var75.i = var72.i | var74.i; + /* 34: storel */ + ptr0[i] = var75; + } + } + +} + +void +compositor_orc_overlay_bgra (guint8 * ORC_RESTRICT d1, int d1_stride, + const guint8 * ORC_RESTRICT s1, int s1_stride, int p1, int n, int m) +{ + OrcExecutor _ex, *ex = &_ex; + static volatile int p_inited = 0; + static OrcCode *c = 0; + void (*func) (OrcExecutor *); + + if (!p_inited) { + orc_once_mutex_lock (); + if (!p_inited) { + OrcProgram *p; + +#if 1 + static const orc_uint8 bc[] = { + 1, 7, 9, 27, 99, 111, 109, 112, 111, 115, 105, 116, 111, 114, 95, 111, + 114, 99, 95, 111, 118, 101, 114, 108, 97, 121, 95, 98, 103, 114, 97, 11, + 4, 4, 12, 4, 4, 14, 4, 255, 255, 255, 255, 14, 4, 0, 0, 0, + 255, 14, 4, 255, 255, 255, 0, 14, 4, 24, 0, 0, 0, 14, 4, 8, + 0, 0, 0, 16, 2, 20, 4, 20, 4, 20, 2, 20, 1, 20, 8, 20, + 8, 20, 8, 20, 4, 20, 8, 20, 8, 113, 32, 4, 126, 33, 32, 19, + 163, 34, 33, 157, 35, 34, 152, 39, 35, 21, 2, 150, 36, 39, 21, 2, + 89, 36, 36, 24, 21, 2, 95, 36, 36, 20, 21, 2, 150, 41, 32, 21, + 2, 89, 41, 41, 36, 115, 39, 16, 21, 2, 150, 37, 39, 21, 2, 98, + 37, 37, 36, 113, 32, 0, 126, 33, 32, 19, 163, 34, 33, 157, 35, 34, + 152, 39, 35, 21, 2, 150, 38, 39, 21, 2, 89, 38, 38, 37, 21, 2, + 80, 38, 38, 21, 2, 150, 40, 32, 21, 2, 89, 40, 40, 38, 21, 2, + 70, 40, 40, 41, 21, 2, 70, 38, 38, 36, 21, 2, 81, 40, 40, 38, + 21, 2, 157, 32, 40, 106, 32, 32, 18, 21, 2, 157, 39, 38, 106, 39, + 39, 17, 123, 32, 32, 39, 128, 0, 32, 2, 0, + }; + p = orc_program_new_from_static_bytecode (bc); + orc_program_set_backup_function (p, _backup_compositor_orc_overlay_bgra); +#else + p = orc_program_new (); + orc_program_set_2d (p); + orc_program_set_name (p, "compositor_orc_overlay_bgra"); + orc_program_set_backup_function (p, _backup_compositor_orc_overlay_bgra); + orc_program_add_destination (p, 4, "d1"); + orc_program_add_source (p, 4, "s1"); + orc_program_add_constant (p, 4, 0xffffffff, "c1"); + orc_program_add_constant (p, 4, 0xff000000, "c2"); + orc_program_add_constant (p, 4, 0x00ffffff, "c3"); + orc_program_add_constant (p, 4, 0x00000018, "c4"); + orc_program_add_constant (p, 4, 0x00000008, "c5"); + orc_program_add_parameter (p, 2, "p1"); + orc_program_add_temporary (p, 4, "t1"); + orc_program_add_temporary (p, 4, "t2"); + orc_program_add_temporary (p, 2, "t3"); + orc_program_add_temporary (p, 1, "t4"); + orc_program_add_temporary (p, 8, "t5"); + orc_program_add_temporary (p, 8, "t6"); + orc_program_add_temporary (p, 8, "t7"); + orc_program_add_temporary (p, 4, "t8"); + orc_program_add_temporary (p, 8, "t9"); + orc_program_add_temporary (p, 8, "t10"); + + orc_program_append_2 (p, "loadl", 0, ORC_VAR_T1, ORC_VAR_S1, ORC_VAR_D1, + ORC_VAR_D1); + orc_program_append_2 (p, "shrul", 0, ORC_VAR_T2, ORC_VAR_T1, ORC_VAR_C4, + ORC_VAR_D1); + orc_program_append_2 (p, "convlw", 0, ORC_VAR_T3, ORC_VAR_T2, ORC_VAR_D1, + ORC_VAR_D1); + orc_program_append_2 (p, "convwb", 0, ORC_VAR_T4, ORC_VAR_T3, ORC_VAR_D1, + ORC_VAR_D1); + orc_program_append_2 (p, "splatbl", 0, ORC_VAR_T8, ORC_VAR_T4, ORC_VAR_D1, + ORC_VAR_D1); + orc_program_append_2 (p, "convubw", 2, ORC_VAR_T5, ORC_VAR_T8, ORC_VAR_D1, + ORC_VAR_D1); + orc_program_append_2 (p, "mullw", 2, ORC_VAR_T5, ORC_VAR_T5, ORC_VAR_P1, + ORC_VAR_D1); + orc_program_append_2 (p, "shruw", 2, ORC_VAR_T5, ORC_VAR_T5, ORC_VAR_C5, + ORC_VAR_D1); + orc_program_append_2 (p, "convubw", 2, ORC_VAR_T10, ORC_VAR_T1, + ORC_VAR_D1, ORC_VAR_D1); + orc_program_append_2 (p, "mullw", 2, ORC_VAR_T10, ORC_VAR_T10, ORC_VAR_T5, + ORC_VAR_D1); + orc_program_append_2 (p, "loadpl", 0, ORC_VAR_T8, ORC_VAR_C1, ORC_VAR_D1, + ORC_VAR_D1); + orc_program_append_2 (p, "convubw", 2, ORC_VAR_T6, ORC_VAR_T8, ORC_VAR_D1, + ORC_VAR_D1); + orc_program_append_2 (p, "subw", 2, ORC_VAR_T6, ORC_VAR_T6, ORC_VAR_T5, + ORC_VAR_D1); + orc_program_append_2 (p, "loadl", 0, ORC_VAR_T1, ORC_VAR_D1, ORC_VAR_D1, + ORC_VAR_D1); + orc_program_append_2 (p, "shrul", 0, ORC_VAR_T2, ORC_VAR_T1, ORC_VAR_C4, + ORC_VAR_D1); + orc_program_append_2 (p, "convlw", 0, ORC_VAR_T3, ORC_VAR_T2, ORC_VAR_D1, + ORC_VAR_D1); + orc_program_append_2 (p, "convwb", 0, ORC_VAR_T4, ORC_VAR_T3, ORC_VAR_D1, + ORC_VAR_D1); + orc_program_append_2 (p, "splatbl", 0, ORC_VAR_T8, ORC_VAR_T4, ORC_VAR_D1, + ORC_VAR_D1); + orc_program_append_2 (p, "convubw", 2, ORC_VAR_T7, ORC_VAR_T8, ORC_VAR_D1, + ORC_VAR_D1); + orc_program_append_2 (p, "mullw", 2, ORC_VAR_T7, ORC_VAR_T7, ORC_VAR_T6, + ORC_VAR_D1); + orc_program_append_2 (p, "div255w", 2, ORC_VAR_T7, ORC_VAR_T7, ORC_VAR_D1, + ORC_VAR_D1); + orc_program_append_2 (p, "convubw", 2, ORC_VAR_T9, ORC_VAR_T1, ORC_VAR_D1, + ORC_VAR_D1); + orc_program_append_2 (p, "mullw", 2, ORC_VAR_T9, ORC_VAR_T9, ORC_VAR_T7, + ORC_VAR_D1); + orc_program_append_2 (p, "addw", 2, ORC_VAR_T9, ORC_VAR_T9, ORC_VAR_T10, + ORC_VAR_D1); + orc_program_append_2 (p, "addw", 2, ORC_VAR_T7, ORC_VAR_T7, ORC_VAR_T5, + ORC_VAR_D1); + orc_program_append_2 (p, "divluw", 2, ORC_VAR_T9, ORC_VAR_T9, ORC_VAR_T7, + ORC_VAR_D1); + orc_program_append_2 (p, "convwb", 2, ORC_VAR_T1, ORC_VAR_T9, ORC_VAR_D1, + ORC_VAR_D1); + orc_program_append_2 (p, "andl", 0, ORC_VAR_T1, ORC_VAR_T1, ORC_VAR_C3, + ORC_VAR_D1); + orc_program_append_2 (p, "convwb", 2, ORC_VAR_T8, ORC_VAR_T7, ORC_VAR_D1, + ORC_VAR_D1); + orc_program_append_2 (p, "andl", 0, ORC_VAR_T8, ORC_VAR_T8, ORC_VAR_C2, + ORC_VAR_D1); + orc_program_append_2 (p, "orl", 0, ORC_VAR_T1, ORC_VAR_T1, ORC_VAR_T8, + ORC_VAR_D1); + orc_program_append_2 (p, "storel", 0, ORC_VAR_D1, ORC_VAR_T1, ORC_VAR_D1, + ORC_VAR_D1); +#endif + + orc_program_compile (p); + c = orc_program_take_code (p); + orc_program_free (p); + } + p_inited = TRUE; + orc_once_mutex_unlock (); + } + ex->arrays[ORC_VAR_A2] = c; + ex->program = 0; + + ex->n = n; + ORC_EXECUTOR_M (ex) = m; + ex->arrays[ORC_VAR_D1] = d1; + ex->params[ORC_VAR_D1] = d1_stride; + ex->arrays[ORC_VAR_S1] = (void *) s1; + ex->params[ORC_VAR_S1] = s1_stride; + ex->params[ORC_VAR_P1] = p1; + + func = c->exec; + func (ex); +} +#endif diff --git a/gst/compositor/compositororc-dist.h b/gst/compositor/compositororc-dist.h new file mode 100644 index 0000000000..907b262b15 --- /dev/null +++ b/gst/compositor/compositororc-dist.h @@ -0,0 +1,96 @@ + +/* autogenerated from compositororc.orc */ + +#ifndef _COMPOSITORORC_H_ +#define _COMPOSITORORC_H_ + +#include + +#ifdef __cplusplus +extern "C" { +#endif + + + +#ifndef _ORC_INTEGER_TYPEDEFS_ +#define _ORC_INTEGER_TYPEDEFS_ +#if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L +#include +typedef int8_t orc_int8; +typedef int16_t orc_int16; +typedef int32_t orc_int32; +typedef int64_t orc_int64; +typedef uint8_t orc_uint8; +typedef uint16_t orc_uint16; +typedef uint32_t orc_uint32; +typedef uint64_t orc_uint64; +#define ORC_UINT64_C(x) UINT64_C(x) +#elif defined(_MSC_VER) +typedef signed __int8 orc_int8; +typedef signed __int16 orc_int16; +typedef signed __int32 orc_int32; +typedef signed __int64 orc_int64; +typedef unsigned __int8 orc_uint8; +typedef unsigned __int16 orc_uint16; +typedef unsigned __int32 orc_uint32; +typedef unsigned __int64 orc_uint64; +#define ORC_UINT64_C(x) (x##Ui64) +#define inline __inline +#else +#include +typedef signed char orc_int8; +typedef short orc_int16; +typedef int orc_int32; +typedef unsigned char orc_uint8; +typedef unsigned short orc_uint16; +typedef unsigned int orc_uint32; +#if INT_MAX == LONG_MAX +typedef long long orc_int64; +typedef unsigned long long orc_uint64; +#define ORC_UINT64_C(x) (x##ULL) +#else +typedef long orc_int64; +typedef unsigned long orc_uint64; +#define ORC_UINT64_C(x) (x##UL) +#endif +#endif +typedef union { orc_int16 i; orc_int8 x2[2]; } orc_union16; +typedef union { orc_int32 i; float f; orc_int16 x2[2]; orc_int8 x4[4]; } orc_union32; +typedef union { orc_int64 i; double f; orc_int32 x2[2]; float x2f[2]; orc_int16 x4[4]; } orc_union64; +#endif +#ifndef ORC_RESTRICT +#if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L +#define ORC_RESTRICT restrict +#elif defined(__GNUC__) && __GNUC__ >= 4 +#define ORC_RESTRICT __restrict__ +#else +#define ORC_RESTRICT +#endif +#endif + +#ifndef ORC_INTERNAL +#if defined(__SUNPRO_C) && (__SUNPRO_C >= 0x590) +#define ORC_INTERNAL __attribute__((visibility("hidden"))) +#elif defined(__SUNPRO_C) && (__SUNPRO_C >= 0x550) +#define ORC_INTERNAL __hidden +#elif defined (__GNUC__) +#define ORC_INTERNAL __attribute__((visibility("hidden"))) +#else +#define ORC_INTERNAL +#endif +#endif + +void compositor_orc_splat_u32 (guint32 * ORC_RESTRICT d1, int p1, int n); +void compositor_orc_memcpy_u32 (guint32 * ORC_RESTRICT d1, const guint32 * ORC_RESTRICT s1, int n); +void compositor_orc_blend_u8 (guint8 * ORC_RESTRICT d1, int d1_stride, const guint8 * ORC_RESTRICT s1, int s1_stride, int p1, int n, int m); +void compositor_orc_blend_argb (guint8 * ORC_RESTRICT d1, int d1_stride, const guint8 * ORC_RESTRICT s1, int s1_stride, int p1, int n, int m); +void compositor_orc_blend_bgra (guint8 * ORC_RESTRICT d1, int d1_stride, const guint8 * ORC_RESTRICT s1, int s1_stride, int p1, int n, int m); +void compositor_orc_overlay_argb (guint8 * ORC_RESTRICT d1, int d1_stride, const guint8 * ORC_RESTRICT s1, int s1_stride, int p1, int n, int m); +void compositor_orc_overlay_bgra (guint8 * ORC_RESTRICT d1, int d1_stride, const guint8 * ORC_RESTRICT s1, int s1_stride, int p1, int n, int m); + +#ifdef __cplusplus +} +#endif + +#endif + diff --git a/gst/compositor/compositororc.orc b/gst/compositor/compositororc.orc new file mode 100644 index 0000000000..5a348b2539 --- /dev/null +++ b/gst/compositor/compositororc.orc @@ -0,0 +1,220 @@ +.function compositor_orc_splat_u32 +.dest 4 d1 guint32 +.param 4 p1 guint32 + +copyl d1, p1 + +.function compositor_orc_memcpy_u32 +.dest 4 d1 guint32 +.source 4 s1 guint32 + +copyl d1, s1 + +.function compositor_orc_blend_u8 +.flags 2d +.dest 1 d1 guint8 +.source 1 s1 guint8 +.param 2 p1 +.temp 2 t1 +.temp 2 t2 +.const 1 c1 8 + +convubw t1, d1 +convubw t2, s1 +subw t2, t2, t1 +mullw t2, t2, p1 +shlw t1, t1, c1 +addw t2, t1, t2 +shruw t2, t2, c1 +convsuswb d1, t2 + + +.function compositor_orc_blend_argb +.flags 2d +.dest 4 d guint8 +.source 4 s guint8 +.param 2 alpha +.temp 4 t +.temp 2 tw +.temp 1 tb +.temp 4 a +.temp 8 d_wide +.temp 8 s_wide +.temp 8 a_wide +.const 4 a_alpha 0x000000ff + +loadl t, s +convlw tw, t +convwb tb, tw +splatbl a, tb +x4 convubw a_wide, a +x4 mullw a_wide, a_wide, alpha +x4 shruw a_wide, a_wide, 8 +x4 convubw s_wide, t +loadl t, d +x4 convubw d_wide, t +x4 subw s_wide, s_wide, d_wide +x4 mullw s_wide, s_wide, a_wide +x4 div255w s_wide, s_wide +x4 addw d_wide, d_wide, s_wide +x4 convwb t, d_wide +orl t, t, a_alpha +storel d, t + +.function compositor_orc_blend_bgra +.flags 2d +.dest 4 d guint8 +.source 4 s guint8 +.param 2 alpha +.temp 4 t +.temp 4 t2 +.temp 2 tw +.temp 1 tb +.temp 4 a +.temp 8 d_wide +.temp 8 s_wide +.temp 8 a_wide +.const 4 a_alpha 0xff000000 + +loadl t, s +shrul t2, t, 24 +convlw tw, t2 +convwb tb, tw +splatbl a, tb +x4 convubw a_wide, a +x4 mullw a_wide, a_wide, alpha +x4 shruw a_wide, a_wide, 8 +x4 convubw s_wide, t +loadl t, d +x4 convubw d_wide, t +x4 subw s_wide, s_wide, d_wide +x4 mullw s_wide, s_wide, a_wide +x4 div255w s_wide, s_wide +x4 addw d_wide, d_wide, s_wide +x4 convwb t, d_wide +orl t, t, a_alpha +storel d, t + + +.function compositor_orc_overlay_argb +.flags 2d +.dest 4 d guint8 +.source 4 s guint8 +.param 2 alpha +.temp 4 t +.temp 2 tw +.temp 1 tb +.temp 8 alpha_s +.temp 8 alpha_s_inv +.temp 8 alpha_d +.temp 4 a +.temp 8 d_wide +.temp 8 s_wide +.const 4 xfs 0xffffffff +.const 4 a_alpha 0x000000ff +.const 4 a_alpha_inv 0xffffff00 + +# calc source alpha as alpha_s = alpha_s * alpha / 256 +loadl t, s +convlw tw, t +convwb tb, tw +splatbl a, tb +x4 convubw alpha_s, a +x4 mullw alpha_s, alpha_s, alpha +x4 shruw alpha_s, alpha_s, 8 +x4 convubw s_wide, t +x4 mullw s_wide, s_wide, alpha_s + +# calc destination alpha as alpha_d = (255-alpha_s) * alpha_d / 255 +loadpl a, xfs +x4 convubw alpha_s_inv, a +x4 subw alpha_s_inv, alpha_s_inv, alpha_s +loadl t, d +convlw tw, t +convwb tb, tw +splatbl a, tb +x4 convubw alpha_d, a +x4 mullw alpha_d, alpha_d, alpha_s_inv +x4 div255w alpha_d, alpha_d +x4 convubw d_wide, t +x4 mullw d_wide, d_wide, alpha_d + +# calc final pixel as pix_d = pix_s*alpha_s + pix_d*alpha_d*(255-alpha_s)/255 +x4 addw d_wide, d_wide, s_wide + +# calc the final destination alpha_d = alpha_s + alpha_d * (255-alpha_s)/255 +x4 addw alpha_d, alpha_d, alpha_s + +# now normalize the pix_d by the final alpha to make it associative +x4 divluw, d_wide, d_wide, alpha_d + +# pack the new alpha into the correct spot +x4 convwb t, d_wide +andl t, t, a_alpha_inv +x4 convwb a, alpha_d +andl a, a, a_alpha +orl t, t, a +storel d, t + +.function compositor_orc_overlay_bgra +.flags 2d +.dest 4 d guint8 +.source 4 s guint8 +.param 2 alpha +.temp 4 t +.temp 4 t2 +.temp 2 tw +.temp 1 tb +.temp 8 alpha_s +.temp 8 alpha_s_inv +.temp 8 alpha_d +.temp 4 a +.temp 8 d_wide +.temp 8 s_wide +.const 4 xfs 0xffffffff +.const 4 a_alpha 0xff000000 +.const 4 a_alpha_inv 0x00ffffff + +# calc source alpha as alpha_s = alpha_s * alpha / 256 +loadl t, s +shrul t2, t, 24 +convlw tw, t2 +convwb tb, tw +splatbl a, tb +x4 convubw alpha_s, a +x4 mullw alpha_s, alpha_s, alpha +x4 shruw alpha_s, alpha_s, 8 +x4 convubw s_wide, t +x4 mullw s_wide, s_wide, alpha_s + +# calc destination alpha as alpha_d = (255-alpha_s) * alpha_d / 255 +loadpl a, xfs +x4 convubw alpha_s_inv, a +x4 subw alpha_s_inv, alpha_s_inv, alpha_s +loadl t, d +shrul t2, t, 24 +convlw tw, t2 +convwb tb, tw +splatbl a, tb +x4 convubw alpha_d, a +x4 mullw alpha_d, alpha_d, alpha_s_inv +x4 div255w alpha_d, alpha_d +x4 convubw d_wide, t +x4 mullw d_wide, d_wide, alpha_d + +# calc final pixel as pix_d = pix_s*alpha_s + pix_d*alpha_d*(255-alpha_s)/255 +x4 addw d_wide, d_wide, s_wide + +# calc the final destination alpha_d = alpha_s + alpha_d * (255-alpha_s)/255 +x4 addw alpha_d, alpha_d, alpha_s + +# now normalize the pix_d by the final alpha to make it associative +x4 divluw, d_wide, d_wide, alpha_d + +# pack the new alpha into the correct spot +x4 convwb t, d_wide +andl t, t, a_alpha_inv +x4 convwb a, alpha_d +andl a, a, a_alpha +orl t, t, a +storel d, t diff --git a/gst/compositor/compositorpad.h b/gst/compositor/compositorpad.h new file mode 100644 index 0000000000..0ba580e5b1 --- /dev/null +++ b/gst/compositor/compositorpad.h @@ -0,0 +1,65 @@ +/* Generic compositor plugin pad + * Copyright (C) 2008 Wim Taymans + * Copyright (C) 2010 Sebastian Dröge + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifndef __GST_COMPOSITOR_PAD_H__ +#define __GST_COMPOSITOR_PAD_H__ + +#include +#include + +G_BEGIN_DECLS + +#define GST_TYPE_COMPOSITOR_PAD (gst_compositor_pad_get_type()) +#define GST_COMPOSITOR_PAD(obj) \ + (G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_COMPOSITOR_PAD, GstCompositorPad)) +#define GST_COMPOSITOR_PAD_CLASS(klass) \ + (G_TYPE_CHECK_CLASS_CAST((klass),GST_TYPE_COMPOSITOR_PAD, GstCompositorPadClass)) +#define GST_IS_COMPOSITOR_PAD(obj) \ + (G_TYPE_CHECK_INSTANCE_TYPE((obj),GST_TYPE_COMPOSITOR_PAD)) +#define GST_IS_COMPOSITOR_PAD_CLASS(klass) \ + (G_TYPE_CHECK_CLASS_TYPE((klass),GST_TYPE_COMPOSITOR_PAD)) + +typedef struct _GstCompositorPad GstCompositorPad; +typedef struct _GstCompositorPadClass GstCompositorPadClass; + +/** + * GstCompositorPad: + * + * The opaque #GstCompositorPad structure. + */ +struct _GstCompositorPad +{ + GstVideoAggregatorPad parent; + + /* properties */ + gint xpos, ypos; + guint zorder; + gdouble alpha; +}; + +struct _GstCompositorPadClass +{ + GstVideoAggregatorPadClass parent_class; +}; + +GType gst_compositor_pad_get_type (void); + +G_END_DECLS +#endif /* __GST_COMPOSITOR_PAD_H__ */ diff --git a/tests/check/elements/compositor.c b/tests/check/elements/compositor.c new file mode 100644 index 0000000000..20b1c3930b --- /dev/null +++ b/tests/check/elements/compositor.c @@ -0,0 +1,1071 @@ +/* GStreamer + * + * unit test for compositor + * + * Copyright (C) <2005> Thomas Vander Stichele + * Copyright (C) <2013> Thibault Saunier + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifdef HAVE_CONFIG_H +# include +#endif + +#ifdef HAVE_VALGRIND +# include +#endif + +#include + +#include +#include +#include + +#define VIDEO_CAPS_STRING \ + "video/x-raw, " \ + "width = (int) 320, " \ + "height = (int) 240, " \ + "framerate = (fraction) 25/1 , " \ + "format = (string) I420" + +static GMainLoop *main_loop; + +/* make sure downstream gets a CAPS event before buffers are sent */ +GST_START_TEST (test_caps) +{ + GstElement *pipeline, *src, *compositor, *sink; + GstStateChangeReturn state_res; + GstCaps *caps; + GstPad *pad; + + /* build pipeline */ + pipeline = gst_pipeline_new ("pipeline"); + + src = gst_element_factory_make ("videotestsrc", "src1"); + compositor = gst_element_factory_make ("compositor", "compositor"); + sink = gst_element_factory_make ("fakesink", "sink"); + gst_bin_add_many (GST_BIN (pipeline), src, compositor, sink, NULL); + + fail_unless (gst_element_link_many (src, compositor, sink, NULL)); + + /* prepare playing */ + state_res = gst_element_set_state (pipeline, GST_STATE_PAUSED); + fail_unless_equals_int (state_res, GST_STATE_CHANGE_ASYNC); + + /* wait for preroll */ + state_res = gst_element_get_state (pipeline, NULL, NULL, GST_CLOCK_TIME_NONE); + fail_unless_equals_int (state_res, GST_STATE_CHANGE_SUCCESS); + + /* check caps on fakesink */ + pad = gst_element_get_static_pad (sink, "sink"); + caps = gst_pad_get_current_caps (pad); + fail_unless (caps != NULL); + gst_caps_unref (caps); + gst_object_unref (pad); + + gst_element_set_state (pipeline, GST_STATE_NULL); + gst_object_unref (pipeline); +} + +GST_END_TEST; + +static void +message_received (GstBus * bus, GstMessage * message, GstPipeline * bin) +{ + GST_INFO ("bus message from \"%" GST_PTR_FORMAT "\": %" GST_PTR_FORMAT, + GST_MESSAGE_SRC (message), message); + + switch (message->type) { + case GST_MESSAGE_EOS: + g_main_loop_quit (main_loop); + break; + case GST_MESSAGE_WARNING:{ + GError *gerror; + gchar *debug; + + gst_message_parse_warning (message, &gerror, &debug); + gst_object_default_error (GST_MESSAGE_SRC (message), gerror, debug); + g_error_free (gerror); + g_free (debug); + break; + } + case GST_MESSAGE_ERROR:{ + GError *gerror; + gchar *debug; + + gst_message_parse_error (message, &gerror, &debug); + gst_object_default_error (GST_MESSAGE_SRC (message), gerror, debug); + g_error_free (gerror); + g_free (debug); + g_main_loop_quit (main_loop); + break; + } + default: + break; + } +} + + +static GstFormat format = GST_FORMAT_UNDEFINED; +static gint64 position = -1; + +static void +test_event_message_received (GstBus * bus, GstMessage * message, + GstPipeline * bin) +{ + GST_INFO ("bus message from \"%" GST_PTR_FORMAT "\": %" GST_PTR_FORMAT, + GST_MESSAGE_SRC (message), message); + + switch (message->type) { + case GST_MESSAGE_SEGMENT_DONE: + gst_message_parse_segment_done (message, &format, &position); + GST_INFO ("received segment_done : %" G_GINT64_FORMAT, position); + g_main_loop_quit (main_loop); + break; + default: + g_assert_not_reached (); + break; + } +} + + +GST_START_TEST (test_event) +{ + GstElement *bin, *src1, *src2, *compositor, *sink; + GstBus *bus; + GstEvent *seek_event; + GstStateChangeReturn state_res; + gboolean res; + GstPad *srcpad, *sinkpad; + GstStreamConsistency *chk_1, *chk_2, *chk_3; + + GST_INFO ("preparing test"); + + /* build pipeline */ + bin = gst_pipeline_new ("pipeline"); + bus = gst_element_get_bus (bin); + gst_bus_add_signal_watch_full (bus, G_PRIORITY_HIGH); + + src1 = gst_element_factory_make ("videotestsrc", "src1"); + src2 = gst_element_factory_make ("videotestsrc", "src2"); + compositor = gst_element_factory_make ("compositor", "compositor"); + sink = gst_element_factory_make ("fakesink", "sink"); + gst_bin_add_many (GST_BIN (bin), src1, src2, compositor, sink, NULL); + + res = gst_element_link (src1, compositor); + fail_unless (res == TRUE, NULL); + res = gst_element_link (src2, compositor); + fail_unless (res == TRUE, NULL); + res = gst_element_link (compositor, sink); + fail_unless (res == TRUE, NULL); + + srcpad = gst_element_get_static_pad (compositor, "src"); + chk_3 = gst_consistency_checker_new (srcpad); + gst_object_unref (srcpad); + + /* create consistency checkers for the pads */ + srcpad = gst_element_get_static_pad (src1, "src"); + chk_1 = gst_consistency_checker_new (srcpad); + sinkpad = gst_pad_get_peer (srcpad); + gst_consistency_checker_add_pad (chk_3, sinkpad); + gst_object_unref (sinkpad); + gst_object_unref (srcpad); + + srcpad = gst_element_get_static_pad (src2, "src"); + chk_2 = gst_consistency_checker_new (srcpad); + sinkpad = gst_pad_get_peer (srcpad); + gst_consistency_checker_add_pad (chk_3, sinkpad); + gst_object_unref (sinkpad); + gst_object_unref (srcpad); + + seek_event = gst_event_new_seek (1.0, GST_FORMAT_TIME, + GST_SEEK_FLAG_SEGMENT | GST_SEEK_FLAG_FLUSH, + GST_SEEK_TYPE_SET, (GstClockTime) 0, + GST_SEEK_TYPE_SET, (GstClockTime) 2 * GST_SECOND); + + format = GST_FORMAT_UNDEFINED; + position = -1; + + main_loop = g_main_loop_new (NULL, FALSE); + g_signal_connect (bus, "message::segment-done", + (GCallback) test_event_message_received, bin); + g_signal_connect (bus, "message::error", (GCallback) message_received, bin); + g_signal_connect (bus, "message::warning", (GCallback) message_received, bin); + g_signal_connect (bus, "message::eos", (GCallback) message_received, bin); + + GST_INFO ("starting test"); + + /* prepare playing */ + state_res = gst_element_set_state (bin, GST_STATE_PAUSED); + ck_assert_int_ne (state_res, GST_STATE_CHANGE_FAILURE); + + /* wait for completion */ + state_res = gst_element_get_state (bin, NULL, NULL, GST_CLOCK_TIME_NONE); + ck_assert_int_ne (state_res, GST_STATE_CHANGE_FAILURE); + + res = gst_element_send_event (bin, seek_event); + fail_unless (res == TRUE, NULL); + + /* run pipeline */ + state_res = gst_element_set_state (bin, GST_STATE_PLAYING); + ck_assert_int_ne (state_res, GST_STATE_CHANGE_FAILURE); + + GST_INFO ("running main loop"); + g_main_loop_run (main_loop); + + state_res = gst_element_set_state (bin, GST_STATE_NULL); + ck_assert_int_ne (state_res, GST_STATE_CHANGE_FAILURE); + + ck_assert_int_eq (position, 2 * GST_SECOND); + + /* cleanup */ + g_main_loop_unref (main_loop); + gst_consistency_checker_free (chk_1); + gst_consistency_checker_free (chk_2); + gst_consistency_checker_free (chk_3); + gst_bus_remove_signal_watch (bus); + gst_object_unref (bus); + gst_object_unref (bin); +} + +GST_END_TEST; + +static guint play_count = 0; +static GstEvent *play_seek_event = NULL; + +static void +test_play_twice_message_received (GstBus * bus, GstMessage * message, + GstPipeline * bin) +{ + gboolean res; + GstStateChangeReturn state_res; + + GST_INFO ("bus message from \"%" GST_PTR_FORMAT "\": %" GST_PTR_FORMAT, + GST_MESSAGE_SRC (message), message); + + switch (message->type) { + case GST_MESSAGE_SEGMENT_DONE: + play_count++; + if (play_count == 1) { + state_res = gst_element_set_state (GST_ELEMENT (bin), GST_STATE_READY); + ck_assert_int_ne (state_res, GST_STATE_CHANGE_FAILURE); + + /* prepare playing again */ + state_res = gst_element_set_state (GST_ELEMENT (bin), GST_STATE_PAUSED); + ck_assert_int_ne (state_res, GST_STATE_CHANGE_FAILURE); + + /* wait for completion */ + state_res = + gst_element_get_state (GST_ELEMENT (bin), NULL, NULL, + GST_CLOCK_TIME_NONE); + ck_assert_int_ne (state_res, GST_STATE_CHANGE_FAILURE); + + res = gst_element_send_event (GST_ELEMENT (bin), + gst_event_ref (play_seek_event)); + fail_unless (res == TRUE, NULL); + + state_res = + gst_element_set_state (GST_ELEMENT (bin), GST_STATE_PLAYING); + ck_assert_int_ne (state_res, GST_STATE_CHANGE_FAILURE); + } else { + g_main_loop_quit (main_loop); + } + break; + default: + g_assert_not_reached (); + break; + } +} + + +GST_START_TEST (test_play_twice) +{ + GstElement *bin, *src1, *src2, *compositor, *sink; + GstBus *bus; + gboolean res; + GstStateChangeReturn state_res; + GstPad *srcpad; + GstStreamConsistency *consist; + + GST_INFO ("preparing test"); + + /* build pipeline */ + bin = gst_pipeline_new ("pipeline"); + bus = gst_element_get_bus (bin); + gst_bus_add_signal_watch_full (bus, G_PRIORITY_HIGH); + + src1 = gst_element_factory_make ("videotestsrc", "src1"); + src2 = gst_element_factory_make ("videotestsrc", "src2"); + compositor = gst_element_factory_make ("compositor", "compositor"); + sink = gst_element_factory_make ("fakesink", "sink"); + gst_bin_add_many (GST_BIN (bin), src1, src2, compositor, sink, NULL); + + res = gst_element_link (src1, compositor); + fail_unless (res == TRUE, NULL); + res = gst_element_link (src2, compositor); + fail_unless (res == TRUE, NULL); + res = gst_element_link (compositor, sink); + fail_unless (res == TRUE, NULL); + + srcpad = gst_element_get_static_pad (compositor, "src"); + consist = gst_consistency_checker_new (srcpad); + gst_object_unref (srcpad); + + play_seek_event = gst_event_new_seek (1.0, GST_FORMAT_TIME, + GST_SEEK_FLAG_SEGMENT | GST_SEEK_FLAG_FLUSH, + GST_SEEK_TYPE_SET, (GstClockTime) 0, + GST_SEEK_TYPE_SET, (GstClockTime) 2 * GST_SECOND); + + play_count = 0; + + main_loop = g_main_loop_new (NULL, FALSE); + g_signal_connect (bus, "message::segment-done", + (GCallback) test_play_twice_message_received, bin); + g_signal_connect (bus, "message::error", (GCallback) message_received, bin); + g_signal_connect (bus, "message::warning", (GCallback) message_received, bin); + g_signal_connect (bus, "message::eos", (GCallback) message_received, bin); + + GST_INFO ("starting test"); + + /* prepare playing */ + state_res = gst_element_set_state (bin, GST_STATE_PAUSED); + ck_assert_int_ne (state_res, GST_STATE_CHANGE_FAILURE); + + /* wait for completion */ + state_res = + gst_element_get_state (GST_ELEMENT (bin), NULL, NULL, + GST_CLOCK_TIME_NONE); + ck_assert_int_ne (state_res, GST_STATE_CHANGE_FAILURE); + + res = gst_element_send_event (bin, gst_event_ref (play_seek_event)); + fail_unless (res == TRUE, NULL); + + GST_INFO ("seeked"); + + /* run pipeline */ + state_res = gst_element_set_state (bin, GST_STATE_PLAYING); + ck_assert_int_ne (state_res, GST_STATE_CHANGE_FAILURE); + + g_main_loop_run (main_loop); + + state_res = gst_element_set_state (bin, GST_STATE_NULL); + ck_assert_int_ne (state_res, GST_STATE_CHANGE_FAILURE); + + ck_assert_int_eq (play_count, 2); + + /* cleanup */ + g_main_loop_unref (main_loop); + gst_consistency_checker_free (consist); + gst_event_ref (play_seek_event); + gst_bus_remove_signal_watch (bus); + gst_object_unref (bus); + gst_object_unref (bin); +} + +GST_END_TEST; + +GST_START_TEST (test_play_twice_then_add_and_play_again) +{ + GstElement *bin, *src1, *src2, *src3, *compositor, *sink; + GstBus *bus; + gboolean res; + GstStateChangeReturn state_res; + gint i; + GstPad *srcpad; + GstStreamConsistency *consist; + + GST_INFO ("preparing test"); + + /* build pipeline */ + bin = gst_pipeline_new ("pipeline"); + bus = gst_element_get_bus (bin); + gst_bus_add_signal_watch_full (bus, G_PRIORITY_HIGH); + + src1 = gst_element_factory_make ("videotestsrc", "src1"); + src2 = gst_element_factory_make ("videotestsrc", "src2"); + compositor = gst_element_factory_make ("compositor", "compositor"); + sink = gst_element_factory_make ("fakesink", "sink"); + gst_bin_add_many (GST_BIN (bin), src1, src2, compositor, sink, NULL); + + srcpad = gst_element_get_static_pad (compositor, "src"); + consist = gst_consistency_checker_new (srcpad); + gst_object_unref (srcpad); + + res = gst_element_link (src1, compositor); + fail_unless (res == TRUE, NULL); + res = gst_element_link (src2, compositor); + fail_unless (res == TRUE, NULL); + res = gst_element_link (compositor, sink); + fail_unless (res == TRUE, NULL); + + play_seek_event = gst_event_new_seek (1.0, GST_FORMAT_TIME, + GST_SEEK_FLAG_SEGMENT | GST_SEEK_FLAG_FLUSH, + GST_SEEK_TYPE_SET, (GstClockTime) 0, + GST_SEEK_TYPE_SET, (GstClockTime) 2 * GST_SECOND); + + main_loop = g_main_loop_new (NULL, FALSE); + g_signal_connect (bus, "message::segment-done", + (GCallback) test_play_twice_message_received, bin); + g_signal_connect (bus, "message::error", (GCallback) message_received, bin); + g_signal_connect (bus, "message::warning", (GCallback) message_received, bin); + g_signal_connect (bus, "message::eos", (GCallback) message_received, bin); + + /* run it twice */ + for (i = 0; i < 2; i++) { + play_count = 0; + + GST_INFO ("starting test-loop %d", i); + + /* prepare playing */ + state_res = gst_element_set_state (bin, GST_STATE_PAUSED); + ck_assert_int_ne (state_res, GST_STATE_CHANGE_FAILURE); + + /* wait for completion */ + state_res = + gst_element_get_state (GST_ELEMENT (bin), NULL, NULL, + GST_CLOCK_TIME_NONE); + ck_assert_int_ne (state_res, GST_STATE_CHANGE_FAILURE); + + res = gst_element_send_event (bin, gst_event_ref (play_seek_event)); + fail_unless (res == TRUE, NULL); + + GST_INFO ("seeked"); + + /* run pipeline */ + state_res = gst_element_set_state (bin, GST_STATE_PLAYING); + ck_assert_int_ne (state_res, GST_STATE_CHANGE_FAILURE); + + g_main_loop_run (main_loop); + + state_res = gst_element_set_state (bin, GST_STATE_READY); + ck_assert_int_ne (state_res, GST_STATE_CHANGE_FAILURE); + + ck_assert_int_eq (play_count, 2); + + /* plug another source */ + if (i == 0) { + src3 = gst_element_factory_make ("videotestsrc", "src3"); + gst_bin_add (GST_BIN (bin), src3); + + res = gst_element_link (src3, compositor); + fail_unless (res == TRUE, NULL); + } + + gst_consistency_checker_reset (consist); + } + + state_res = gst_element_set_state (bin, GST_STATE_NULL); + ck_assert_int_ne (state_res, GST_STATE_CHANGE_FAILURE); + + /* cleanup */ + g_main_loop_unref (main_loop); + gst_event_ref (play_seek_event); + gst_consistency_checker_free (consist); + gst_bus_remove_signal_watch (bus); + gst_object_unref (bus); + gst_object_unref (bin); +} + +GST_END_TEST; + +/* check if adding pads work as expected */ +GST_START_TEST (test_add_pad) +{ + GstElement *bin, *src1, *src2, *compositor, *sink; + GstBus *bus; + GstPad *srcpad; + gboolean res; + GstStateChangeReturn state_res; + + GST_INFO ("preparing test"); + + /* build pipeline */ + bin = gst_pipeline_new ("pipeline"); + bus = gst_element_get_bus (bin); + gst_bus_add_signal_watch_full (bus, G_PRIORITY_HIGH); + + src1 = gst_element_factory_make ("videotestsrc", "src1"); + g_object_set (src1, "num-buffers", 4, NULL); + src2 = gst_element_factory_make ("videotestsrc", "src2"); + /* one buffer less, we connect with 1 buffer of delay */ + g_object_set (src2, "num-buffers", 3, NULL); + compositor = gst_element_factory_make ("compositor", "compositor"); + sink = gst_element_factory_make ("fakesink", "sink"); + gst_bin_add_many (GST_BIN (bin), src1, compositor, sink, NULL); + + res = gst_element_link (src1, compositor); + fail_unless (res == TRUE, NULL); + res = gst_element_link (compositor, sink); + fail_unless (res == TRUE, NULL); + + srcpad = gst_element_get_static_pad (compositor, "src"); + gst_object_unref (srcpad); + + main_loop = g_main_loop_new (NULL, FALSE); + g_signal_connect (bus, "message::segment-done", (GCallback) message_received, + bin); + g_signal_connect (bus, "message::error", (GCallback) message_received, bin); + g_signal_connect (bus, "message::warning", (GCallback) message_received, bin); + g_signal_connect (bus, "message::eos", (GCallback) message_received, bin); + + GST_INFO ("starting test"); + + /* prepare playing */ + state_res = gst_element_set_state (bin, GST_STATE_PAUSED); + ck_assert_int_ne (state_res, GST_STATE_CHANGE_FAILURE); + + /* wait for completion */ + state_res = + gst_element_get_state (GST_ELEMENT (bin), NULL, NULL, + GST_CLOCK_TIME_NONE); + ck_assert_int_ne (state_res, GST_STATE_CHANGE_FAILURE); + + /* add other element */ + gst_bin_add_many (GST_BIN (bin), src2, NULL); + + /* now link the second element */ + res = gst_element_link (src2, compositor); + fail_unless (res == TRUE, NULL); + + /* set to PAUSED as well */ + state_res = gst_element_set_state (src2, GST_STATE_PAUSED); + ck_assert_int_ne (state_res, GST_STATE_CHANGE_FAILURE); + + /* now play all */ + state_res = gst_element_set_state (bin, GST_STATE_PLAYING); + ck_assert_int_ne (state_res, GST_STATE_CHANGE_FAILURE); + + g_main_loop_run (main_loop); + + state_res = gst_element_set_state (bin, GST_STATE_NULL); + ck_assert_int_ne (state_res, GST_STATE_CHANGE_FAILURE); + + /* cleanup */ + g_main_loop_unref (main_loop); + gst_bus_remove_signal_watch (bus); + gst_object_unref (bus); + gst_object_unref (bin); +} + +GST_END_TEST; + +/* check if removing pads work as expected */ +GST_START_TEST (test_remove_pad) +{ + GstElement *bin, *src, *compositor, *sink; + GstBus *bus; + GstPad *pad, *srcpad; + gboolean res; + GstStateChangeReturn state_res; + + GST_INFO ("preparing test"); + + /* build pipeline */ + bin = gst_pipeline_new ("pipeline"); + bus = gst_element_get_bus (bin); + gst_bus_add_signal_watch_full (bus, G_PRIORITY_HIGH); + + src = gst_element_factory_make ("videotestsrc", "src"); + g_object_set (src, "num-buffers", 4, NULL); + compositor = gst_element_factory_make ("compositor", "compositor"); + sink = gst_element_factory_make ("fakesink", "sink"); + gst_bin_add_many (GST_BIN (bin), src, compositor, sink, NULL); + + res = gst_element_link (src, compositor); + fail_unless (res == TRUE, NULL); + res = gst_element_link (compositor, sink); + fail_unless (res == TRUE, NULL); + + /* create an unconnected sinkpad in compositor */ + pad = gst_element_get_request_pad (compositor, "sink_%u"); + fail_if (pad == NULL, NULL); + + srcpad = gst_element_get_static_pad (compositor, "src"); + gst_object_unref (srcpad); + + main_loop = g_main_loop_new (NULL, FALSE); + g_signal_connect (bus, "message::segment-done", (GCallback) message_received, + bin); + g_signal_connect (bus, "message::error", (GCallback) message_received, bin); + g_signal_connect (bus, "message::warning", (GCallback) message_received, bin); + g_signal_connect (bus, "message::eos", (GCallback) message_received, bin); + + GST_INFO ("starting test"); + + /* prepare playing, this will not preroll as compositor is waiting + * on the unconnected sinkpad. */ + state_res = gst_element_set_state (bin, GST_STATE_PAUSED); + ck_assert_int_ne (state_res, GST_STATE_CHANGE_FAILURE); + + /* wait for completion for one second, will return ASYNC */ + state_res = gst_element_get_state (GST_ELEMENT (bin), NULL, NULL, GST_SECOND); + ck_assert_int_eq (state_res, GST_STATE_CHANGE_ASYNC); + + /* get rid of the pad now, compositor should stop waiting on it and + * continue the preroll */ + gst_element_release_request_pad (compositor, pad); + gst_object_unref (pad); + + /* wait for completion, should work now */ + state_res = + gst_element_get_state (GST_ELEMENT (bin), NULL, NULL, + GST_CLOCK_TIME_NONE); + ck_assert_int_ne (state_res, GST_STATE_CHANGE_FAILURE); + + /* now play all */ + state_res = gst_element_set_state (bin, GST_STATE_PLAYING); + ck_assert_int_ne (state_res, GST_STATE_CHANGE_FAILURE); + + g_main_loop_run (main_loop); + + state_res = gst_element_set_state (bin, GST_STATE_NULL); + ck_assert_int_ne (state_res, GST_STATE_CHANGE_FAILURE); + + /* cleanup */ + g_main_loop_unref (main_loop); + gst_bus_remove_signal_watch (bus); + gst_object_unref (G_OBJECT (bus)); + gst_object_unref (G_OBJECT (bin)); +} + +GST_END_TEST; + + +static GstBuffer *handoff_buffer = NULL; + +static gboolean +_quit (GMainLoop * ml) +{ + g_main_loop_quit (ml); + + return G_SOURCE_REMOVE; +} + +static void +handoff_buffer_cb (GstElement * fakesink, GstBuffer * buffer, GstPad * pad, + gpointer user_data) +{ + GST_DEBUG ("got buffer %p", buffer); + gst_buffer_replace (&handoff_buffer, buffer); + + if (main_loop) + g_idle_add ((GSourceFunc) _quit, main_loop); +} + +/* check if clipping works as expected */ +GST_START_TEST (test_clip) +{ + GstSegment segment; + GstElement *bin, *compositor, *sink; + GstBus *bus; + GstPad *sinkpad; + gboolean res; + GstStateChangeReturn state_res; + GstFlowReturn ret; + GstEvent *event; + GstBuffer *buffer; + GstCaps *caps; + GMainLoop *local_mainloop; + + GST_INFO ("preparing test"); + + local_mainloop = g_main_loop_new (NULL, FALSE); + main_loop = NULL; + + /* build pipeline */ + bin = gst_pipeline_new ("pipeline"); + bus = gst_element_get_bus (bin); + gst_bus_add_signal_watch_full (bus, G_PRIORITY_HIGH); + + g_signal_connect (bus, "message::error", (GCallback) message_received, bin); + g_signal_connect (bus, "message::warning", (GCallback) message_received, bin); + g_signal_connect (bus, "message::eos", (GCallback) message_received, bin); + + /* just an compositor and a fakesink */ + compositor = gst_element_factory_make ("compositor", "compositor"); + sink = gst_element_factory_make ("fakesink", "sink"); + g_object_set (sink, "signal-handoffs", TRUE, NULL); + g_signal_connect (sink, "handoff", (GCallback) handoff_buffer_cb, NULL); + gst_bin_add_many (GST_BIN (bin), compositor, sink, NULL); + + res = gst_element_link (compositor, sink); + fail_unless (res == TRUE, NULL); + + /* set to playing */ + state_res = gst_element_set_state (bin, GST_STATE_PLAYING); + ck_assert_int_ne (state_res, GST_STATE_CHANGE_FAILURE); + + /* create an unconnected sinkpad in compositor, should also automatically activate + * the pad */ + sinkpad = gst_element_get_request_pad (compositor, "sink_%u"); + fail_if (sinkpad == NULL, NULL); + + gst_pad_send_event (sinkpad, gst_event_new_stream_start ("test")); + + caps = gst_caps_from_string (VIDEO_CAPS_STRING); + + gst_pad_set_caps (sinkpad, caps); + gst_caps_unref (caps); + + /* send segment to compositor */ + gst_segment_init (&segment, GST_FORMAT_TIME); + segment.start = GST_SECOND; + segment.stop = 2 * GST_SECOND; + segment.time = 0; + event = gst_event_new_segment (&segment); + gst_pad_send_event (sinkpad, event); + + /* should be clipped and ok */ + buffer = gst_buffer_new_and_alloc (115200); + GST_BUFFER_TIMESTAMP (buffer) = 0; + GST_BUFFER_DURATION (buffer) = 250 * GST_MSECOND; + GST_DEBUG ("pushing buffer %p", buffer); + ret = gst_pad_chain (sinkpad, buffer); + ck_assert_int_eq (ret, GST_FLOW_OK); + fail_unless (handoff_buffer == NULL); + + /* should be partially clipped */ + buffer = gst_buffer_new_and_alloc (115200); + GST_BUFFER_TIMESTAMP (buffer) = 900 * GST_MSECOND; + GST_BUFFER_DURATION (buffer) = 250 * GST_MSECOND; + GST_DEBUG ("pushing buffer %p", buffer); + + main_loop = local_mainloop; + ret = gst_pad_chain (sinkpad, buffer); + ck_assert_int_eq (ret, GST_FLOW_OK); + g_main_loop_run (main_loop); + gst_buffer_replace (&handoff_buffer, NULL); + + /* should not be clipped */ + buffer = gst_buffer_new_and_alloc (115200); + GST_BUFFER_TIMESTAMP (buffer) = 1 * GST_SECOND; + GST_BUFFER_DURATION (buffer) = 250 * GST_MSECOND; + GST_DEBUG ("pushing buffer %p", buffer); + ret = gst_pad_chain (sinkpad, buffer); + g_main_loop_run (main_loop); + main_loop = NULL; + ck_assert_int_eq (ret, GST_FLOW_OK); + fail_unless (handoff_buffer != NULL); + gst_buffer_replace (&handoff_buffer, NULL); + + /* should be clipped and ok */ + buffer = gst_buffer_new_and_alloc (115200); + GST_BUFFER_TIMESTAMP (buffer) = 2 * GST_SECOND; + GST_BUFFER_DURATION (buffer) = 250 * GST_MSECOND; + GST_DEBUG ("pushing buffer %p", buffer); + ret = gst_pad_chain (sinkpad, buffer); + ck_assert_int_eq (ret, GST_FLOW_OK); + fail_unless (handoff_buffer == NULL); + + gst_object_unref (sinkpad); + gst_element_set_state (bin, GST_STATE_NULL); + g_main_loop_unref (local_mainloop); + gst_bus_remove_signal_watch (bus); + gst_object_unref (bus); + gst_object_unref (bin); +} + +GST_END_TEST; + +GST_START_TEST (test_duration_is_max) +{ + GstElement *bin, *src[3], *compositor, *sink; + GstStateChangeReturn state_res; + GstFormat format = GST_FORMAT_TIME; + gboolean res; + gint64 duration; + + GST_INFO ("preparing test"); + + /* build pipeline */ + bin = gst_pipeline_new ("pipeline"); + + /* 3 sources, an compositor and a fakesink */ + src[0] = gst_element_factory_make ("videotestsrc", NULL); + src[1] = gst_element_factory_make ("videotestsrc", NULL); + src[2] = gst_element_factory_make ("videotestsrc", NULL); + compositor = gst_element_factory_make ("compositor", "compositor"); + sink = gst_element_factory_make ("fakesink", "sink"); + gst_bin_add_many (GST_BIN (bin), src[0], src[1], src[2], compositor, sink, + NULL); + + gst_element_link (src[0], compositor); + gst_element_link (src[1], compositor); + gst_element_link (src[2], compositor); + gst_element_link (compositor, sink); + + /* irks, duration is reset on basesrc */ + state_res = gst_element_set_state (bin, GST_STATE_PAUSED); + fail_unless (state_res != GST_STATE_CHANGE_FAILURE, NULL); + + /* set durations on src */ + GST_BASE_SRC (src[0])->segment.duration = 1000; + GST_BASE_SRC (src[1])->segment.duration = 3000; + GST_BASE_SRC (src[2])->segment.duration = 2000; + + /* set to playing */ + state_res = gst_element_set_state (bin, GST_STATE_PLAYING); + fail_unless (state_res != GST_STATE_CHANGE_FAILURE, NULL); + + /* wait for completion */ + state_res = + gst_element_get_state (GST_ELEMENT (bin), NULL, NULL, + GST_CLOCK_TIME_NONE); + fail_unless (state_res != GST_STATE_CHANGE_FAILURE, NULL); + + res = gst_element_query_duration (GST_ELEMENT (bin), format, &duration); + fail_unless (res, NULL); + + ck_assert_int_eq (duration, 3000); + + gst_element_set_state (bin, GST_STATE_NULL); + gst_object_unref (bin); +} + +GST_END_TEST; + +GST_START_TEST (test_duration_unknown_overrides) +{ + GstElement *bin, *src[3], *compositor, *sink; + GstStateChangeReturn state_res; + GstFormat format = GST_FORMAT_TIME; + gboolean res; + gint64 duration; + + GST_INFO ("preparing test"); + + /* build pipeline */ + bin = gst_pipeline_new ("pipeline"); + + /* 3 sources, an compositor and a fakesink */ + src[0] = gst_element_factory_make ("videotestsrc", NULL); + src[1] = gst_element_factory_make ("videotestsrc", NULL); + src[2] = gst_element_factory_make ("videotestsrc", NULL); + compositor = gst_element_factory_make ("compositor", "compositor"); + sink = gst_element_factory_make ("fakesink", "sink"); + gst_bin_add_many (GST_BIN (bin), src[0], src[1], src[2], compositor, sink, + NULL); + + gst_element_link (src[0], compositor); + gst_element_link (src[1], compositor); + gst_element_link (src[2], compositor); + gst_element_link (compositor, sink); + + /* irks, duration is reset on basesrc */ + state_res = gst_element_set_state (bin, GST_STATE_PAUSED); + fail_unless (state_res != GST_STATE_CHANGE_FAILURE, NULL); + + /* set durations on src */ + GST_BASE_SRC (src[0])->segment.duration = GST_CLOCK_TIME_NONE; + GST_BASE_SRC (src[1])->segment.duration = 3000; + GST_BASE_SRC (src[2])->segment.duration = 2000; + + /* set to playing */ + state_res = gst_element_set_state (bin, GST_STATE_PLAYING); + fail_unless (state_res != GST_STATE_CHANGE_FAILURE, NULL); + + /* wait for completion */ + state_res = + gst_element_get_state (GST_ELEMENT (bin), NULL, NULL, + GST_CLOCK_TIME_NONE); + fail_unless (state_res != GST_STATE_CHANGE_FAILURE, NULL); + + res = gst_element_query_duration (GST_ELEMENT (bin), format, &duration); + fail_unless (res, NULL); + + ck_assert_int_eq (duration, GST_CLOCK_TIME_NONE); + + gst_element_set_state (bin, GST_STATE_NULL); + gst_object_unref (bin); +} + +GST_END_TEST; + + +static gboolean looped = FALSE; + +static void +loop_segment_done (GstBus * bus, GstMessage * message, GstElement * bin) +{ + GST_INFO ("bus message from \"%" GST_PTR_FORMAT "\": %" GST_PTR_FORMAT, + GST_MESSAGE_SRC (message), message); + + if (looped) { + g_main_loop_quit (main_loop); + } else { + GstEvent *seek_event; + gboolean res; + + seek_event = gst_event_new_seek (1.0, GST_FORMAT_TIME, + GST_SEEK_FLAG_SEGMENT, + GST_SEEK_TYPE_SET, (GstClockTime) 0, + GST_SEEK_TYPE_SET, (GstClockTime) 1 * GST_SECOND); + + res = gst_element_send_event (bin, seek_event); + fail_unless (res == TRUE, NULL); + looped = TRUE; + } +} + +GST_START_TEST (test_loop) +{ + GstElement *bin, *src1, *src2, *compositor, *sink; + GstBus *bus; + GstEvent *seek_event; + GstStateChangeReturn state_res; + gboolean res; + + GST_INFO ("preparing test"); + + /* build pipeline */ + bin = gst_pipeline_new ("pipeline"); + bus = gst_element_get_bus (bin); + gst_bus_add_signal_watch_full (bus, G_PRIORITY_HIGH); + + src1 = gst_element_factory_make ("videotestsrc", "src1"); + src2 = gst_element_factory_make ("videotestsrc", "src2"); + compositor = gst_element_factory_make ("compositor", "compositor"); + sink = gst_element_factory_make ("fakesink", "sink"); + gst_bin_add_many (GST_BIN (bin), src1, src2, compositor, sink, NULL); + + res = gst_element_link (src1, compositor); + fail_unless (res == TRUE, NULL); + res = gst_element_link (src2, compositor); + fail_unless (res == TRUE, NULL); + res = gst_element_link (compositor, sink); + fail_unless (res == TRUE, NULL); + + seek_event = gst_event_new_seek (1.0, GST_FORMAT_TIME, + GST_SEEK_FLAG_SEGMENT | GST_SEEK_FLAG_FLUSH, + GST_SEEK_TYPE_SET, (GstClockTime) 0, GST_SEEK_TYPE_SET, + (GstClockTime) 2 * GST_SECOND); + + main_loop = g_main_loop_new (NULL, FALSE); + g_signal_connect (bus, "message::segment-done", + (GCallback) loop_segment_done, bin); + g_signal_connect (bus, "message::error", (GCallback) message_received, bin); + g_signal_connect (bus, "message::warning", (GCallback) message_received, bin); + g_signal_connect (bus, "message::eos", (GCallback) message_received, bin); + + GST_INFO ("starting test"); + + /* prepare playing */ + state_res = gst_element_set_state (bin, GST_STATE_PAUSED); + ck_assert_int_ne (state_res, GST_STATE_CHANGE_FAILURE); + + /* wait for completion */ + state_res = + gst_element_get_state (GST_ELEMENT (bin), NULL, NULL, + GST_CLOCK_TIME_NONE); + ck_assert_int_ne (state_res, GST_STATE_CHANGE_FAILURE); + + res = gst_element_send_event (bin, seek_event); + fail_unless (res == TRUE, NULL); + + /* run pipeline */ + state_res = gst_element_set_state (bin, GST_STATE_PLAYING); + ck_assert_int_ne (state_res, GST_STATE_CHANGE_FAILURE); + + GST_INFO ("running main loop"); + g_main_loop_run (main_loop); + + state_res = gst_element_set_state (bin, GST_STATE_NULL); + + /* cleanup */ + g_main_loop_unref (main_loop); + gst_bus_remove_signal_watch (bus); + gst_object_unref (bus); + gst_object_unref (bin); +} + +GST_END_TEST; + +GST_START_TEST (test_flush_start_flush_stop) +{ + GstPadTemplate *sink_template; + GstPad *sinkpad1, *sinkpad2, *compositor_src; + GstElement *compositor; + + GST_INFO ("preparing test"); + + /* build pipeline */ + compositor = gst_element_factory_make ("compositor", "compositor"); + + sink_template = + gst_element_class_get_pad_template (GST_ELEMENT_GET_CLASS (compositor), + "sink_%u"); + fail_unless (GST_IS_PAD_TEMPLATE (sink_template)); + sinkpad1 = gst_element_request_pad (compositor, sink_template, NULL, NULL); + sinkpad2 = gst_element_request_pad (compositor, sink_template, NULL, NULL); + gst_object_unref (sinkpad2); + + gst_element_set_state (compositor, GST_STATE_PLAYING); + fail_unless (gst_element_get_state (compositor, NULL, NULL, + GST_CLOCK_TIME_NONE) == GST_STATE_CHANGE_SUCCESS); + + compositor_src = gst_element_get_static_pad (compositor, "src"); + fail_if (GST_PAD_IS_FLUSHING (compositor_src)); + gst_pad_send_event (sinkpad1, gst_event_new_flush_start ()); + fail_if (GST_PAD_IS_FLUSHING (compositor_src)); + fail_unless (GST_PAD_IS_FLUSHING (sinkpad1)); + gst_pad_send_event (sinkpad1, gst_event_new_flush_stop (TRUE)); + fail_if (GST_PAD_IS_FLUSHING (compositor_src)); + fail_if (GST_PAD_IS_FLUSHING (sinkpad1)); + gst_object_unref (compositor_src); + + /* cleanup */ + gst_element_set_state (compositor, GST_STATE_NULL); + gst_object_unref (compositor); +} + +GST_END_TEST; + + +static Suite * +compositor_suite (void) +{ + Suite *s = suite_create ("compositor"); + TCase *tc_chain = tcase_create ("general"); + + suite_add_tcase (s, tc_chain); + tcase_add_test (tc_chain, test_caps); + tcase_add_test (tc_chain, test_event); + tcase_add_test (tc_chain, test_play_twice); + tcase_add_test (tc_chain, test_play_twice_then_add_and_play_again); + tcase_add_test (tc_chain, test_add_pad); + tcase_add_test (tc_chain, test_remove_pad); + tcase_add_test (tc_chain, test_clip); + tcase_add_test (tc_chain, test_duration_is_max); + tcase_add_test (tc_chain, test_duration_unknown_overrides); + tcase_add_test (tc_chain, test_loop); + tcase_add_test (tc_chain, test_flush_start_flush_stop); + + /* Use a longer timeout */ +#ifdef HAVE_VALGRIND + if (RUNNING_ON_VALGRIND) { + tcase_set_timeout (tc_chain, 5 * 60); + } else +#endif + { + /* this is shorter than the default 60 seconds?! (tpm) */ + /* tcase_set_timeout (tc_chain, 6); */ + } + + return s; +} + +GST_CHECK_MAIN (compositor); From 1f1283b4d8bdd71e4ee70ab339bccde3525214da Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Fri, 20 Jun 2014 11:10:45 +0200 Subject: [PATCH 012/381] gl:glvideomixer: Add the Compositor in the element metadata class So it is possible to pick one compositing element from the registry --- ext/gl/gstglvideomixer.c | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/ext/gl/gstglvideomixer.c b/ext/gl/gstglvideomixer.c index b2ff4008f9..5a5d7a2312 100644 --- a/ext/gl/gstglvideomixer.c +++ b/ext/gl/gstglvideomixer.c @@ -257,7 +257,7 @@ gst_gl_video_mixer_class_init (GstGLVideoMixerClass * klass) gobject_class->get_property = gst_gl_video_mixer_get_property; gst_element_class_set_metadata (element_class, "OpenGL video_mixer", - "Filter/Effect/Video", "OpenGL video_mixer", + "Filter/Effect/Video/Compositor", "OpenGL video_mixer", "Julien Isorce "); GST_GL_MIXER_CLASS (klass)->set_caps = gst_gl_video_mixer_init_shader; @@ -401,10 +401,8 @@ gst_gl_video_mixer_callback (gpointer stuff) continue; } pad = (GstGLVideoMixerPad *) frame->pad; - in_width = - GST_VIDEO_INFO_WIDTH (&GST_VIDEO_AGGREGATOR_PAD (pad)->info); - in_height = - GST_VIDEO_INFO_HEIGHT (&GST_VIDEO_AGGREGATOR_PAD (pad)->info); + in_width = GST_VIDEO_INFO_WIDTH (&GST_VIDEO_AGGREGATOR_PAD (pad)->info); + in_height = GST_VIDEO_INFO_HEIGHT (&GST_VIDEO_AGGREGATOR_PAD (pad)->info); if (!frame->texture || in_width <= 0 || in_height <= 0) { GST_DEBUG ("skipping texture:%u frame:%p width:%u height %u", From 9e0c81cb625bdf7d49b2dfdab9fe7830b0e030e0 Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Fri, 20 Jun 2014 22:02:07 +0200 Subject: [PATCH 013/381] libs: videoaggregato: Do not import videoconvert.h in gstvideoaggregatorpad.h + Add a Private structure to the GstVideoAggregatorPad + Add some padding --- gst-libs/gst/video/gstvideoaggregator.c | 39 +++++++++++++++++-------- 1 file changed, 27 insertions(+), 12 deletions(-) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index 06d80e19d6..b0ea776544 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -40,9 +40,10 @@ #include +#include "videoconvert.h" + #include "gstvideoaggregator.h" #include "gstvideoaggregatorpad.h" -#include "videoconvert.h" GST_DEBUG_CATEGORY_STATIC (gst_videoaggregator_debug); #define GST_CAT_DEFAULT gst_videoaggregator_debug @@ -61,6 +62,13 @@ enum PROP_PAD_ZORDER, }; + +struct _GstVideoAggregatorPadPrivate +{ + /* Converter, if NULL no conversion is done */ + VideoConvert *convert; +}; + G_DEFINE_TYPE (GstVideoAggregatorPad, gst_videoaggregator_pad, GST_TYPE_AGGREGATOR_PAD); @@ -130,9 +138,9 @@ gst_videoaggregator_pad_finalize (GObject * o) { GstVideoAggregatorPad *vaggpad = GST_VIDEO_AGGREGATOR_PAD (o); - if (vaggpad->convert) - videoconvert_convert_free (vaggpad->convert); - vaggpad->convert = NULL; + if (vaggpad->priv->convert) + videoconvert_convert_free (vaggpad->priv->convert); + vaggpad->priv->convert = NULL; G_OBJECT_CLASS (gst_videoaggregator_pad_parent_class)->dispose (o); } @@ -152,17 +160,24 @@ gst_videoaggregator_pad_class_init (GstVideoAggregatorPadClass * klass) 0, 10000, DEFAULT_PAD_ZORDER, G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); + g_type_class_add_private (klass, sizeof (GstVideoAggregatorPadPrivate)); + aggpadclass->flush = GST_DEBUG_FUNCPTR (_flush_pad); } static void gst_videoaggregator_pad_init (GstVideoAggregatorPad * vaggpad) { + vaggpad->priv = + G_TYPE_INSTANCE_GET_PRIVATE (vaggpad, GST_TYPE_VIDEO_AGGREGATOR_PAD, + GstVideoAggregatorPadPrivate); + vaggpad->zorder = DEFAULT_PAD_ZORDER; - vaggpad->convert = NULL; vaggpad->need_conversion_update = FALSE; vaggpad->aggregated_frame = NULL; vaggpad->converted_buffer = NULL; + + vaggpad->priv->convert = NULL; } /********************************* @@ -417,10 +432,10 @@ gst_videoaggregator_update_converters (GstVideoAggregator * vagg) if (GST_VIDEO_INFO_FORMAT (&pad->info) == GST_VIDEO_FORMAT_UNKNOWN) continue; - if (pad->convert) - videoconvert_convert_free (pad->convert); + if (pad->priv->convert) + videoconvert_convert_free (pad->priv->convert); - pad->convert = NULL; + pad->priv->convert = NULL; colorimetry = gst_video_colorimetry_to_string (&(pad->info.colorimetry)); chroma = gst_video_chroma_to_string (pad->info.chroma_site); @@ -431,9 +446,9 @@ gst_videoaggregator_update_converters (GstVideoAggregator * vagg) GST_DEBUG_OBJECT (pad, "This pad will be converted from %d to %d", GST_VIDEO_INFO_FORMAT (&pad->info), GST_VIDEO_INFO_FORMAT (&best_info)); - pad->convert = videoconvert_convert_new (&pad->info, &best_info); + pad->priv->convert = videoconvert_convert_new (&pad->info, &best_info); pad->need_conversion_update = TRUE; - if (!pad->convert) { + if (!pad->priv->convert) { g_free (colorimetry); g_free (best_colorimetry); GST_WARNING ("No path found for conversion"); @@ -1016,7 +1031,7 @@ prepare_frames (GstVideoAggregator * vagg, GstVideoAggregatorPad * pad) GST_WARNING_OBJECT (vagg, "Could not map input buffer"); } - if (pad->convert) { + if (pad->priv->convert) { gint converted_size; /* We wait until here to set the conversion infos, in case vagg->info changed */ @@ -1040,7 +1055,7 @@ prepare_frames (GstVideoAggregator * vagg, GstVideoAggregatorPad * pad) return FALSE; } - videoconvert_convert_convert (pad->convert, converted_frame, frame); + videoconvert_convert_convert (pad->priv->convert, converted_frame, frame); pad->converted_buffer = converted_buf; gst_video_frame_unmap (frame); } else { From 6be7e91e4a658004317bc9211dc10df0476d3170 Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Sat, 21 Jun 2014 16:52:51 +0200 Subject: [PATCH 014/381] libs:video: Properly declare APIs as UNSTABLE --- gst-libs/gst/video/gstvideoaggregator.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/gst-libs/gst/video/gstvideoaggregator.h b/gst-libs/gst/video/gstvideoaggregator.h index 9fa5071ae6..b9fd53164c 100644 --- a/gst-libs/gst/video/gstvideoaggregator.h +++ b/gst-libs/gst/video/gstvideoaggregator.h @@ -21,6 +21,11 @@ #ifndef __GST_VIDEO_AGGREGATOR_H__ #define __GST_VIDEO_AGGREGATOR_H__ +#ifndef GST_USE_UNSTABLE_API +#warning "The Video library from gst-plugins-bad is unstable API and may change in future." +#warning "You can define GST_USE_UNSTABLE_API to avoid this warning." +#endif + #include #include #include From 776b461eff4970afc3ae2f2bfa8e91d44f8b137a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Sun, 22 Jun 2014 19:22:28 +0200 Subject: [PATCH 015/381] Release 1.3.3 --- gst/compositor/compositororc-dist.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gst/compositor/compositororc-dist.c b/gst/compositor/compositororc-dist.c index 04d97dd243..db71b932e9 100644 --- a/gst/compositor/compositororc-dist.c +++ b/gst/compositor/compositororc-dist.c @@ -133,8 +133,8 @@ void compositor_orc_overlay_bgra (guint8 * ORC_RESTRICT d1, int d1_stride, #define ORC_CLAMP_UW(x) ORC_CLAMP(x,ORC_UW_MIN,ORC_UW_MAX) #define ORC_CLAMP_SL(x) ORC_CLAMP(x,ORC_SL_MIN,ORC_SL_MAX) #define ORC_CLAMP_UL(x) ORC_CLAMP(x,ORC_UL_MIN,ORC_UL_MAX) -#define ORC_SWAP_W(x) ((((x)&0xff)<<8) | (((x)&0xff00)>>8)) -#define ORC_SWAP_L(x) ((((x)&0xff)<<24) | (((x)&0xff00)<<8) | (((x)&0xff0000)>>8) | (((x)&0xff000000)>>24)) +#define ORC_SWAP_W(x) ((((x)&0xffU)<<8) | (((x)&0xff00U)>>8)) +#define ORC_SWAP_L(x) ((((x)&0xffU)<<24) | (((x)&0xff00U)<<8) | (((x)&0xff0000U)>>8) | (((x)&0xff000000U)>>24)) #define ORC_SWAP_Q(x) ((((x)&ORC_UINT64_C(0xff))<<56) | (((x)&ORC_UINT64_C(0xff00))<<40) | (((x)&ORC_UINT64_C(0xff0000))<<24) | (((x)&ORC_UINT64_C(0xff000000))<<8) | (((x)&ORC_UINT64_C(0xff00000000))>>8) | (((x)&ORC_UINT64_C(0xff0000000000))>>24) | (((x)&ORC_UINT64_C(0xff000000000000))>>40) | (((x)&ORC_UINT64_C(0xff00000000000000))>>56)) #define ORC_PTR_OFFSET(ptr,offset) ((void *)(((unsigned char *)(ptr)) + (offset))) #define ORC_DENORMAL(x) ((x) & ((((x)&0x7f800000) == 0) ? 0xff800000 : 0xffffffff)) From 5ae625a333fb3e6bca7bc04d9f33d8495aadecd0 Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Mon, 23 Jun 2014 22:36:23 +1000 Subject: [PATCH 016/381] videoaggregator: fix up the parent chaining for dispose and finalize --- gst-libs/gst/video/gstvideoaggregator.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index b0ea776544..583e540b20 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -142,7 +142,7 @@ gst_videoaggregator_pad_finalize (GObject * o) videoconvert_convert_free (vaggpad->priv->convert); vaggpad->priv->convert = NULL; - G_OBJECT_CLASS (gst_videoaggregator_pad_parent_class)->dispose (o); + G_OBJECT_CLASS (gst_videoaggregator_pad_parent_class)->finalize (o); } static void @@ -1854,6 +1854,8 @@ gst_videoaggregator_dispose (GObject * o) GstVideoAggregator *vagg = GST_VIDEO_AGGREGATOR (o); gst_caps_replace (&vagg->priv->current_caps, NULL); + + G_OBJECT_CLASS (gst_videoaggregator_parent_class)->dispose (o); } static void From fe79c90fc162158c38827b04bc188d83ae7ad2a4 Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Mon, 23 Jun 2014 22:40:23 +1000 Subject: [PATCH 017/381] videoaggregator: don't clobber already heap allocated video frame CID # 1223440 --- gst-libs/gst/video/gstvideoaggregator.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index 583e540b20..6010a52f55 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -1011,7 +1011,7 @@ prepare_frames (GstVideoAggregator * vagg, GstVideoAggregatorPad * pad) GstClockTime timestamp; gint64 stream_time; GstSegment *seg; - GstVideoFrame *converted_frame = g_slice_new0 (GstVideoFrame); + GstVideoFrame *converted_frame; GstBuffer *converted_buf = NULL; GstVideoFrame *frame = g_slice_new0 (GstVideoFrame); @@ -1034,6 +1034,8 @@ prepare_frames (GstVideoAggregator * vagg, GstVideoAggregatorPad * pad) if (pad->priv->convert) { gint converted_size; + converted_frame = g_slice_new0 (GstVideoFrame); + /* We wait until here to set the conversion infos, in case vagg->info changed */ if (pad->need_conversion_update) { pad->conversion_info = vagg->info; From 6c4be30245b85e124bb232964c7538a2708a0967 Mon Sep 17 00:00:00 2001 From: Edward Hervey Date: Tue, 24 Jun 2014 08:01:21 +0200 Subject: [PATCH 018/381] compositor: Fix Makefile CFLAGS/LIBADD ordering We want to use the libraries from -bad if/when present --- gst/compositor/Makefile.am | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/gst/compositor/Makefile.am b/gst/compositor/Makefile.am index a91f0fa4f2..eee77d3d08 100644 --- a/gst/compositor/Makefile.am +++ b/gst/compositor/Makefile.am @@ -10,14 +10,16 @@ libgstcompositor_la_SOURCES = \ nodist_libgstcompositor_la_SOURCES = $(ORC_NODIST_SOURCES) -libgstcompositor_la_CFLAGS = $(GST_PLUGINS_BASE_CFLAGS) \ +libgstcompositor_la_CFLAGS = \ -I$(top_srcdir)/gst-libs \ -I$(top_builddir)/gst-libs \ + $(GST_PLUGINS_BASE_CFLAGS) \ $(GST_BASE_CFLAGS) $(GST_CFLAGS) $(ORC_CFLAGS) -libgstcompositor_la_LIBADD = $(GST_PLUGINS_BASE_LIBS) \ - -lgstvideo-@GST_API_VERSION@ \ +libgstcompositor_la_LIBADD = \ $(top_builddir)/gst-libs/gst/base/libgstbadbase-$(GST_API_VERSION).la \ $(top_builddir)/gst-libs/gst/video/libgstbadvideo-$(GST_API_VERSION).la \ + $(GST_PLUGINS_BASE_LIBS) \ + -lgstvideo-@GST_API_VERSION@ \ $(GST_BASE_LIBS) $(GST_LIBS) $(ORC_LIBS) $(LIBM) libgstcompositor_la_LDFLAGS = $(GST_PLUGIN_LDFLAGS) libgstcompositor_la_LIBTOOLFLAGS = $(GST_PLUGIN_LIBTOOLFLAGS) From 0646c3397057392676a8b20bdb95e5e574b5cd37 Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Wed, 25 Jun 2014 10:18:48 +1000 Subject: [PATCH 019/381] glvideomixer: don't clobber already allocated shader --- ext/gl/gstglvideomixer.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ext/gl/gstglvideomixer.c b/ext/gl/gstglvideomixer.c index 5a5d7a2312..1d0299cf0c 100644 --- a/ext/gl/gstglvideomixer.c +++ b/ext/gl/gstglvideomixer.c @@ -315,6 +315,9 @@ gst_gl_video_mixer_init_shader (GstGLMixer * mixer, GstCaps * outcaps) { GstGLVideoMixer *video_mixer = GST_GL_VIDEO_MIXER (mixer); + if (video_mixer->shader) + gst_gl_context_del_shader (mixer->context, video_mixer->shader); + return gst_gl_context_gen_shader (mixer->context, video_mixer_v_src, video_mixer_f_src, &video_mixer->shader); } From 7f2000a7a526d6f23039974c0c2c2eab6b416a5c Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Wed, 25 Jun 2014 12:00:34 +1000 Subject: [PATCH 020/381] glvideomixer: bas output width/height on the pad properties Allows automatic negotiation of the size in the following case: gst-launch-1.0 glvideomixer name=m sink_0::xpos=0 sink_1::xpos=320 ! glimagesink \ videotestsrc ! m. \ videotestsrc pattern=1 ! m. https://bugzilla.gnome.org/show_bug.cgi?id=731878 --- ext/gl/gstglvideomixer.c | 51 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 50 insertions(+), 1 deletion(-) diff --git a/ext/gl/gstglvideomixer.c b/ext/gl/gstglvideomixer.c index 1d0299cf0c..7b3904e64d 100644 --- a/ext/gl/gstglvideomixer.c +++ b/ext/gl/gstglvideomixer.c @@ -60,6 +60,7 @@ static void gst_gl_video_mixer_set_property (GObject * object, guint prop_id, static void gst_gl_video_mixer_get_property (GObject * object, guint prop_id, GValue * value, GParamSpec * pspec); +static gboolean _update_info (GstVideoAggregator * vagg, GstVideoInfo * info); static void gst_gl_video_mixer_reset (GstGLMixer * mixer); static gboolean gst_gl_video_mixer_init_shader (GstGLMixer * mixer, GstCaps * outcaps); @@ -249,6 +250,7 @@ gst_gl_video_mixer_class_init (GstGLVideoMixerClass * klass) GObjectClass *gobject_class; GstElementClass *element_class; GstAggregatorClass *agg_class = (GstAggregatorClass *) klass; + GstVideoAggregatorClass *vagg_class = (GstVideoAggregatorClass *) klass; gobject_class = (GObjectClass *) klass; element_class = GST_ELEMENT_CLASS (klass); @@ -265,8 +267,9 @@ gst_gl_video_mixer_class_init (GstGLVideoMixerClass * klass) GST_GL_MIXER_CLASS (klass)->process_textures = gst_gl_video_mixer_process_textures; - agg_class->sinkpads_type = GST_TYPE_GL_VIDEO_MIXER_PAD; + vagg_class->update_info = _update_info; + agg_class->sinkpads_type = GST_TYPE_GL_VIDEO_MIXER_PAD; } static void @@ -298,6 +301,52 @@ gst_gl_video_mixer_get_property (GObject * object, guint prop_id, } } +static gboolean +_update_info (GstVideoAggregator * vagg, GstVideoInfo * info) +{ + GList *l; + gint best_width = -1, best_height = -1; + gboolean ret = FALSE; + + GST_OBJECT_LOCK (vagg); + for (l = GST_ELEMENT (vagg)->sinkpads; l; l = l->next) { + GstVideoAggregatorPad *vaggpad = l->data; + GstGLVideoMixerPad *mixer_pad = GST_GL_VIDEO_MIXER_PAD (vaggpad); + gint this_width, this_height; + gint width, height; + + if (mixer_pad->width > 0) + width = mixer_pad->width; + else + width = GST_VIDEO_INFO_WIDTH (&vaggpad->info); + + if (mixer_pad->height > 0) + height = mixer_pad->height; + else + height = GST_VIDEO_INFO_HEIGHT (&vaggpad->info); + + if (width == 0 || height == 0) + continue; + + this_width = width + MAX (mixer_pad->xpos, 0); + this_height = height + MAX (mixer_pad->ypos, 0); + + if (best_width < this_width) + best_width = this_width; + if (best_height < this_height) + best_height = this_height; + } + GST_OBJECT_UNLOCK (vagg); + + if (best_width > 0 && best_height > 0) { + gst_video_info_set_format (info, GST_VIDEO_INFO_FORMAT (info), + best_width, best_height); + ret = TRUE; + } + + return ret; +} + static void gst_gl_video_mixer_reset (GstGLMixer * mixer) { From b0a0974ea7192f495a3d88833bc46d1080aaf593 Mon Sep 17 00:00:00 2001 From: Julien Isorce Date: Sun, 22 Jun 2014 13:14:27 +0100 Subject: [PATCH 021/381] gl: enable glvideomixer on GLES2 --- ext/gl/gstglvideomixer.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ext/gl/gstglvideomixer.c b/ext/gl/gstglvideomixer.c index 7b3904e64d..7dcb9367a6 100644 --- a/ext/gl/gstglvideomixer.c +++ b/ext/gl/gstglvideomixer.c @@ -81,6 +81,9 @@ static const gchar *video_mixer_v_src = /* fragment source */ static const gchar *video_mixer_f_src = + "#ifdef GL_ES\n" + "precision mediump float;\n" + "#endif\n" "uniform sampler2D texture; \n" "uniform float alpha;\n" "varying vec2 v_texCoord; \n" From b1c00adfd74de4da492bc32ea342429380ff7696 Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Fri, 27 Jun 2014 00:09:08 +1000 Subject: [PATCH 022/381] videoaggregator: fix a refcount error when keeping the buffer We take a ref on the pad's buffer at the beginning so we need to unref when we are done in all cases. --- gst-libs/gst/video/gstvideoaggregator.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index 6010a52f55..2bc7e44f8c 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -806,6 +806,7 @@ gst_videoaggregator_reset (GstVideoAggregator * vagg) GstVideoAggregatorPad *p = l->data; gst_buffer_replace (&p->buffer, NULL); + gst_buffer_replace (&p->queued, NULL); p->start_time = -1; p->end_time = -1; @@ -959,6 +960,7 @@ gst_videoaggregator_fill_queues (GstVideoAggregator * vagg, } else if (start_time >= output_end_time) { GST_DEBUG_OBJECT (pad, "Keeping buffer until %" GST_TIME_FORMAT, GST_TIME_ARGS (start_time)); + gst_buffer_unref (buf); eos = FALSE; } else { GST_DEBUG_OBJECT (pad, "Too old buffer -- dropping"); From 5d0564eb4e68180bc42e7dd195123d2d9e381745 Mon Sep 17 00:00:00 2001 From: Thiago Santos Date: Sat, 28 Jun 2014 09:43:48 -0300 Subject: [PATCH 023/381] compositor: tests: Fix pad leak Remember to unref requested pad --- tests/check/elements/compositor.c | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/check/elements/compositor.c b/tests/check/elements/compositor.c index 20b1c3930b..9ef52baeb0 100644 --- a/tests/check/elements/compositor.c +++ b/tests/check/elements/compositor.c @@ -1029,6 +1029,7 @@ GST_START_TEST (test_flush_start_flush_stop) /* cleanup */ gst_element_set_state (compositor, GST_STATE_NULL); + gst_object_unref (sinkpad1); gst_object_unref (compositor); } From 97cf5acb209a29f7154261e80bc0028b29d7c31d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Sat, 28 Jun 2014 17:01:52 +0200 Subject: [PATCH 024/381] badvideo: Rename videoconvert functions to prevent conflicts with static linking https://bugzilla.gnome.org/show_bug.cgi?id=728443 --- gst-libs/gst/video/gstvideoaggregator.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index 2bc7e44f8c..0563a07b3f 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -139,7 +139,7 @@ gst_videoaggregator_pad_finalize (GObject * o) GstVideoAggregatorPad *vaggpad = GST_VIDEO_AGGREGATOR_PAD (o); if (vaggpad->priv->convert) - videoconvert_convert_free (vaggpad->priv->convert); + badvideoconvert_convert_free (vaggpad->priv->convert); vaggpad->priv->convert = NULL; G_OBJECT_CLASS (gst_videoaggregator_pad_parent_class)->finalize (o); @@ -433,7 +433,7 @@ gst_videoaggregator_update_converters (GstVideoAggregator * vagg) continue; if (pad->priv->convert) - videoconvert_convert_free (pad->priv->convert); + badvideoconvert_convert_free (pad->priv->convert); pad->priv->convert = NULL; @@ -446,7 +446,7 @@ gst_videoaggregator_update_converters (GstVideoAggregator * vagg) GST_DEBUG_OBJECT (pad, "This pad will be converted from %d to %d", GST_VIDEO_INFO_FORMAT (&pad->info), GST_VIDEO_INFO_FORMAT (&best_info)); - pad->priv->convert = videoconvert_convert_new (&pad->info, &best_info); + pad->priv->convert = badvideoconvert_convert_new (&pad->info, &best_info); pad->need_conversion_update = TRUE; if (!pad->priv->convert) { g_free (colorimetry); @@ -1059,7 +1059,8 @@ prepare_frames (GstVideoAggregator * vagg, GstVideoAggregatorPad * pad) return FALSE; } - videoconvert_convert_convert (pad->priv->convert, converted_frame, frame); + badvideoconvert_convert_convert (pad->priv->convert, converted_frame, + frame); pad->converted_buffer = converted_buf; gst_video_frame_unmap (frame); } else { From 771ed7e429a8dfa3f841101f5902f0fb070d744a Mon Sep 17 00:00:00 2001 From: Vincent Penquerc'h Date: Tue, 1 Jul 2014 12:52:39 +0100 Subject: [PATCH 025/381] videoaggregator: reset QoS on segment event https://bugzilla.gnome.org/show_bug.cgi?id=732540 --- gst-libs/gst/video/gstvideoaggregator.c | 1 + 1 file changed, 1 insertion(+) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index 0563a07b3f..e3d4f809bd 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -1630,6 +1630,7 @@ gst_videoaggregator_sink_event (GstAggregator * agg, GstAggregatorPad * bpad, gst_event_copy_segment (event, &seg); g_assert (seg.format == GST_FORMAT_TIME); + gst_videoaggregator_reset_qos (vagg); break; } default: From 27ad5ef8d60b5e5a8db258608bb4891d546eb627 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim-Philipp=20M=C3=BCller?= Date: Sun, 6 Jul 2014 22:16:48 +0100 Subject: [PATCH 026/381] videoaggregator: fix broken locking in update_src_caps function We would unlock an already-unlocked mutex that we never re-locked. https://bugzilla.gnome.org/show_bug.cgi?id=732750 --- gst-libs/gst/video/gstvideoaggregator.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index e3d4f809bd..82b041fc8e 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -555,7 +555,6 @@ gst_videoaggregator_update_src_caps (GstVideoAggregator * vagg) best_fps_d = fps_d; } } - GST_OBJECT_UNLOCK (vagg); if (best_fps_n <= 0 || best_fps_d <= 0 || best_fps == 0.0) { best_fps_n = 25; @@ -583,6 +582,8 @@ gst_videoaggregator_update_src_caps (GstVideoAggregator * vagg) info.par_n = GST_VIDEO_INFO_PAR_N (&vagg->info); info.par_d = GST_VIDEO_INFO_PAR_D (&vagg->info); + GST_OBJECT_UNLOCK (vagg); + if (vagg_klass->update_info) { if (!vagg_klass->update_info (vagg, &info)) { ret = FALSE; @@ -594,6 +595,9 @@ gst_videoaggregator_update_src_caps (GstVideoAggregator * vagg) caps = gst_video_info_to_caps (&info); peercaps = gst_pad_peer_query_caps (agg->srcpad, NULL); + + GST_OBJECT_LOCK (vagg); + if (peercaps) { GstCaps *tmp; From 4e60b291f8d4dc5ed29b34299cfa39ebd8cf07bf Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Sun, 6 Jul 2014 23:30:53 +0200 Subject: [PATCH 027/381] videoaggregator: Fix some more the locking logic in update_src_caps We need the GST_OBJECT_LOCK only to iterate the sinkpads, nothing else. https://bugzilla.gnome.org/show_bug.cgi?id=732750 --- gst-libs/gst/video/gstvideoaggregator.c | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index 82b041fc8e..09dcbb71dd 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -555,6 +555,7 @@ gst_videoaggregator_update_src_caps (GstVideoAggregator * vagg) best_fps_d = fps_d; } } + GST_OBJECT_UNLOCK (vagg); if (best_fps_n <= 0 || best_fps_d <= 0 || best_fps == 0.0) { best_fps_n = 25; @@ -582,8 +583,6 @@ gst_videoaggregator_update_src_caps (GstVideoAggregator * vagg) info.par_n = GST_VIDEO_INFO_PAR_N (&vagg->info); info.par_d = GST_VIDEO_INFO_PAR_D (&vagg->info); - GST_OBJECT_UNLOCK (vagg); - if (vagg_klass->update_info) { if (!vagg_klass->update_info (vagg, &info)) { ret = FALSE; @@ -595,9 +594,6 @@ gst_videoaggregator_update_src_caps (GstVideoAggregator * vagg) caps = gst_video_info_to_caps (&info); peercaps = gst_pad_peer_query_caps (agg->srcpad, NULL); - - GST_OBJECT_LOCK (vagg); - if (peercaps) { GstCaps *tmp; @@ -614,7 +610,6 @@ gst_videoaggregator_update_src_caps (GstVideoAggregator * vagg) GST_DEBUG_OBJECT (vagg, "empty caps"); ret = FALSE; GST_VIDEO_AGGREGATOR_UNLOCK (vagg); - GST_OBJECT_UNLOCK (vagg); goto done; } @@ -634,7 +629,6 @@ gst_videoaggregator_update_src_caps (GstVideoAggregator * vagg) caps = gst_video_info_to_caps (&info); GST_VIDEO_AGGREGATOR_UNLOCK (vagg); - GST_OBJECT_UNLOCK (vagg); if (gst_videoaggregator_src_setcaps (vagg, caps)) { if (vagg_klass->negotiated_caps) From 8c7916d4eca9822130c13f252c5ac53db00fec3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Fri, 11 Jul 2014 09:41:05 +0200 Subject: [PATCH 028/381] gl: Move GstGLMixer to the plugin for now It depends on GstAggregator and we don't want to install headers for that yet. https://bugzilla.gnome.org/show_bug.cgi?id=732207 --- ext/gl/gstglmixer.c | 1055 ++++++++++++++++++++++++++++++++++++++ ext/gl/gstglmixer.h | 95 ++++ ext/gl/gstglmosaic.h | 2 +- ext/gl/gstglvideomixer.h | 4 +- 4 files changed, 1153 insertions(+), 3 deletions(-) create mode 100644 ext/gl/gstglmixer.c create mode 100644 ext/gl/gstglmixer.h diff --git a/ext/gl/gstglmixer.c b/ext/gl/gstglmixer.c new file mode 100644 index 0000000000..626eb967c7 --- /dev/null +++ b/ext/gl/gstglmixer.c @@ -0,0 +1,1055 @@ +/* Generic video mixer plugin + * + * GStreamer + * Copyright (C) 2009 Julien Isorce + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include + +#ifdef HAVE_STDLIB_H +#include +#endif +#ifdef HAVE_STRING_H +#include +#endif + +#include "gstglmixer.h" + +#define gst_gl_mixer_parent_class parent_class +G_DEFINE_ABSTRACT_TYPE (GstGLMixer, gst_gl_mixer, GST_TYPE_VIDEO_AGGREGATOR); +static gboolean gst_gl_mixer_do_bufferpool (GstGLMixer * mix, + GstCaps * outcaps); + + +#define GST_CAT_DEFAULT gst_gl_mixer_debug +GST_DEBUG_CATEGORY (gst_gl_mixer_debug); + +static void gst_gl_mixer_pad_get_property (GObject * object, guint prop_id, + GValue * value, GParamSpec * pspec); +static void gst_gl_mixer_pad_set_property (GObject * object, guint prop_id, + const GValue * value, GParamSpec * pspec); + +static void gst_gl_mixer_set_context (GstElement * element, + GstContext * context); + +enum +{ + PROP_PAD_0 +}; + +#define GST_GL_MIXER_GET_PRIVATE(obj) \ + (G_TYPE_INSTANCE_GET_PRIVATE ((obj), GST_TYPE_GL_MIXER, GstGLMixerPrivate)) + +struct _GstGLMixerPrivate +{ + gboolean negotiated; + + GstBufferPool *pool; + gboolean pool_active; + GstAllocator *allocator; + GstAllocationParams params; + GstQuery *query; +}; + +G_DEFINE_TYPE (GstGLMixerPad, gst_gl_mixer_pad, GST_TYPE_VIDEO_AGGREGATOR_PAD); + +static void +gst_gl_mixer_pad_class_init (GstGLMixerPadClass * klass) +{ + GObjectClass *gobject_class = (GObjectClass *) klass; + + gobject_class->set_property = gst_gl_mixer_pad_set_property; + gobject_class->get_property = gst_gl_mixer_pad_get_property; +} + +static void +gst_gl_mixer_pad_get_property (GObject * object, guint prop_id, + GValue * value, GParamSpec * pspec) +{ + switch (prop_id) { + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static void +gst_gl_mixer_pad_set_property (GObject * object, guint prop_id, + const GValue * value, GParamSpec * pspec) +{ + switch (prop_id) { + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static gboolean +_negotiated_caps (GstVideoAggregator * vagg, GstCaps * caps) +{ + GstGLMixer *mix = GST_GL_MIXER (vagg); + gboolean ret = gst_gl_mixer_do_bufferpool (mix, caps); + + mix->priv->negotiated = ret; + + return ret; +} + +static gboolean +gst_gl_mixer_propose_allocation (GstGLMixer * mix, + GstQuery * decide_query, GstQuery * query) +{ + GstBufferPool *pool; + GstStructure *config; + GstCaps *caps; + guint size = 0; + gboolean need_pool; + GError *error = NULL; + GstStructure *gl_context; + gchar *platform, *gl_apis; + gpointer handle; + GstAllocator *allocator = NULL; + GstAllocationParams params; + + gst_query_parse_allocation (query, &caps, &need_pool); + + if (caps == NULL) + goto no_caps; + + if ((pool = mix->priv->pool)) + gst_object_ref (pool); + + if (pool != NULL) { + GstCaps *pcaps; + + /* we had a pool, check caps */ + GST_DEBUG_OBJECT (mix, "check existing pool caps"); + config = gst_buffer_pool_get_config (pool); + gst_buffer_pool_config_get_params (config, &pcaps, &size, NULL, NULL); + + if (!gst_caps_is_equal (caps, pcaps)) { + GST_DEBUG_OBJECT (mix, "pool has different caps"); + /* different caps, we can't use this pool */ + gst_object_unref (pool); + pool = NULL; + } + gst_structure_free (config); + } + + if (!gst_gl_ensure_display (mix, &mix->display)) + return FALSE; + + if (!mix->context) { + mix->context = gst_gl_context_new (mix->display); + if (!gst_gl_context_create (mix->context, NULL, &error)) + goto context_error; + } + + if (pool == NULL && need_pool) { + GstVideoInfo info; + + if (!gst_video_info_from_caps (&info, caps)) + goto invalid_caps; + + GST_DEBUG_OBJECT (mix, "create new pool"); + pool = gst_gl_buffer_pool_new (mix->context); + + /* the normal size of a frame */ + size = info.size; + + config = gst_buffer_pool_get_config (pool); + gst_buffer_pool_config_set_params (config, caps, size, 0, 0); + if (!gst_buffer_pool_set_config (pool, config)) + goto config_failed; + } + + if (pool) { + gst_query_add_allocation_pool (query, pool, size, 1, 0); + gst_object_unref (pool); + } + + /* we also support various metadata */ + gst_query_add_allocation_meta (query, GST_VIDEO_META_API_TYPE, 0); + + gl_apis = gst_gl_api_to_string (gst_gl_context_get_gl_api (mix->context)); + platform = + gst_gl_platform_to_string (gst_gl_context_get_gl_platform (mix->context)); + handle = (gpointer) gst_gl_context_get_gl_context (mix->context); + + gl_context = + gst_structure_new ("GstVideoGLTextureUploadMeta", "gst.gl.GstGLContext", + GST_GL_TYPE_CONTEXT, mix->context, "gst.gl.context.handle", + G_TYPE_POINTER, handle, "gst.gl.context.type", G_TYPE_STRING, platform, + "gst.gl.context.apis", G_TYPE_STRING, gl_apis, NULL); + gst_query_add_allocation_meta (query, + GST_VIDEO_GL_TEXTURE_UPLOAD_META_API_TYPE, gl_context); + + g_free (gl_apis); + g_free (platform); + gst_structure_free (gl_context); + + gst_allocation_params_init (¶ms); + + allocator = gst_allocator_find (GST_GL_MEMORY_ALLOCATOR); + gst_query_add_allocation_param (query, allocator, ¶ms); + gst_object_unref (allocator); + + return TRUE; + + /* ERRORS */ +no_caps: + { + GST_DEBUG_OBJECT (mix, "no caps specified"); + return FALSE; + } +invalid_caps: + { + GST_DEBUG_OBJECT (mix, "invalid caps specified"); + return FALSE; + } +config_failed: + { + GST_DEBUG_OBJECT (mix, "failed setting config"); + return FALSE; + } +context_error: + { + GST_ELEMENT_ERROR (mix, RESOURCE, NOT_FOUND, ("%s", error->message), + (NULL)); + return FALSE; + } +} + +static gboolean +gst_gl_mixer_sink_query (GstAggregator * agg, GstAggregatorPad * bpad, + GstQuery * query) +{ + gboolean ret = FALSE; + GstGLMixer *mix = GST_GL_MIXER (agg); + + GST_TRACE ("QUERY %" GST_PTR_FORMAT, query); + + switch (GST_QUERY_TYPE (query)) { + case GST_QUERY_ALLOCATION: + { + GstQuery *decide_query = NULL; + gboolean negotiated; + + GST_OBJECT_LOCK (mix); + if (G_UNLIKELY (!(negotiated = mix->priv->negotiated))) { + GST_DEBUG_OBJECT (mix, + "not negotiated yet, can't answer ALLOCATION query"); + GST_OBJECT_UNLOCK (mix); + return FALSE; + } + if ((decide_query = mix->priv->query)) + gst_query_ref (decide_query); + GST_OBJECT_UNLOCK (mix); + + GST_DEBUG_OBJECT (mix, + "calling propose allocation with query %" GST_PTR_FORMAT, + decide_query); + + /* pass the query to the propose_allocation vmethod if any */ + ret = gst_gl_mixer_propose_allocation (mix, decide_query, query); + + if (decide_query) + gst_query_unref (decide_query); + + GST_DEBUG_OBJECT (mix, "ALLOCATION ret %d, %" GST_PTR_FORMAT, ret, query); + break; + } + case GST_QUERY_CONTEXT: + { + ret = gst_gl_handle_context_query ((GstElement *) mix, query, + &mix->display); + break; + } + default: + ret = GST_AGGREGATOR_CLASS (parent_class)->sink_query (agg, bpad, query); + break; + } + + return ret; +} + +static void +gst_gl_mixer_pad_init (GstGLMixerPad * mixerpad) +{ +} + +/* GLMixer signals and args */ +enum +{ + /* FILL ME */ + LAST_SIGNAL +}; + +enum +{ + PROP_0 +}; + +static GstStaticPadTemplate src_factory = GST_STATIC_PAD_TEMPLATE ("src", + GST_PAD_SRC, + GST_PAD_ALWAYS, + GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES + (GST_CAPS_FEATURE_MEMORY_GL_MEMORY, + "RGBA") "; " + GST_VIDEO_CAPS_MAKE_WITH_FEATURES + (GST_CAPS_FEATURE_META_GST_VIDEO_GL_TEXTURE_UPLOAD_META, + "RGBA") + "; " GST_VIDEO_CAPS_MAKE (GST_GL_COLOR_CONVERT_FORMATS)) + ); + +static GstStaticPadTemplate sink_factory = GST_STATIC_PAD_TEMPLATE ("sink_%u", + GST_PAD_SINK, + GST_PAD_REQUEST, + GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES + (GST_CAPS_FEATURE_MEMORY_GL_MEMORY, + "RGBA") "; " + GST_VIDEO_CAPS_MAKE_WITH_FEATURES + (GST_CAPS_FEATURE_META_GST_VIDEO_GL_TEXTURE_UPLOAD_META, + "RGBA") + "; " GST_VIDEO_CAPS_MAKE (GST_GL_COLOR_CONVERT_FORMATS)) + ); + +static gboolean gst_gl_mixer_src_query (GstAggregator * agg, GstQuery * query); +static GstFlowReturn +gst_gl_mixer_get_output_buffer (GstVideoAggregator * videoaggregator, + GstBuffer ** outbuf); +static gboolean +gst_gl_mixer_src_activate_mode (GstAggregator * aggregator, GstPadMode mode, + gboolean active); +static gboolean gst_gl_mixer_stop (GstAggregator * agg); +static gboolean gst_gl_mixer_start (GstAggregator * agg); + +static GstFlowReturn +gst_gl_mixer_aggregate_frames (GstVideoAggregator * vagg, + GstBuffer * outbuffer); + +static void gst_gl_mixer_set_property (GObject * object, guint prop_id, + const GValue * value, GParamSpec * pspec); +static void gst_gl_mixer_get_property (GObject * object, guint prop_id, + GValue * value, GParamSpec * pspec); + +static gboolean gst_gl_mixer_decide_allocation (GstGLMixer * mix, + GstQuery * query); +static gboolean gst_gl_mixer_set_allocation (GstGLMixer * mix, + GstBufferPool * pool, GstAllocator * allocator, + GstAllocationParams * params, GstQuery * query); + +static void gst_gl_mixer_finalize (GObject * object); + +static void +gst_gl_mixer_class_init (GstGLMixerClass * klass) +{ + GObjectClass *gobject_class; + GstElementClass *element_class; + + GstVideoAggregatorClass *videoaggregator_class = + (GstVideoAggregatorClass *) klass; + GstAggregatorClass *agg_class = (GstAggregatorClass *) klass; + + GST_DEBUG_CATEGORY_INIT (GST_CAT_DEFAULT, "glmixer", 0, "opengl mixer"); + + gobject_class = (GObjectClass *) klass; + element_class = GST_ELEMENT_CLASS (klass); + + g_type_class_add_private (klass, sizeof (GstGLMixerPrivate)); + + gobject_class->finalize = GST_DEBUG_FUNCPTR (gst_gl_mixer_finalize); + + gobject_class->get_property = gst_gl_mixer_get_property; + gobject_class->set_property = gst_gl_mixer_set_property; + + gst_element_class_add_pad_template (element_class, + gst_static_pad_template_get (&src_factory)); + gst_element_class_add_pad_template (element_class, + gst_static_pad_template_get (&sink_factory)); + + element_class->set_context = GST_DEBUG_FUNCPTR (gst_gl_mixer_set_context); + + agg_class->sinkpads_type = GST_TYPE_GL_MIXER_PAD; + agg_class->sink_query = gst_gl_mixer_sink_query; + agg_class->src_query = gst_gl_mixer_src_query; + agg_class->src_activate = gst_gl_mixer_src_activate_mode; + agg_class->stop = gst_gl_mixer_stop; + agg_class->start = gst_gl_mixer_start; + + videoaggregator_class->disable_frame_conversion = TRUE; + videoaggregator_class->aggregate_frames = gst_gl_mixer_aggregate_frames; + videoaggregator_class->get_output_buffer = gst_gl_mixer_get_output_buffer; + videoaggregator_class->negotiated_caps = _negotiated_caps; + + + /* Register the pad class */ + g_type_class_ref (GST_TYPE_GL_MIXER_PAD); + + klass->set_caps = NULL; + +} + +static void +gst_gl_mixer_reset (GstGLMixer * mix) +{ + /* clean up collect data */ + mix->priv->negotiated = FALSE; +} + +static void +gst_gl_mixer_init (GstGLMixer * mix) +{ + mix->priv = GST_GL_MIXER_GET_PRIVATE (mix); + mix->array_buffers = 0; + mix->display = NULL; + mix->fbo = 0; + mix->depthbuffer = 0; + + /* initialize variables */ + gst_gl_mixer_reset (mix); +} + +static void +gst_gl_mixer_finalize (GObject * object) +{ + G_OBJECT_CLASS (parent_class)->finalize (object); +} + +static void +gst_gl_mixer_set_context (GstElement * element, GstContext * context) +{ + GstGLMixer *mix = GST_GL_MIXER (element); + + gst_gl_handle_set_context (element, context, &mix->display); +} + +static gboolean +gst_gl_mixer_activate (GstGLMixer * mix, gboolean active) +{ + gboolean result = TRUE; + + if (active) { + if (!gst_gl_ensure_display (mix, &mix->display)) + result = FALSE; + } + + return result; +} + +static gboolean +gst_gl_mixer_src_activate_mode (GstAggregator * aggregator, GstPadMode mode, + gboolean active) +{ + GstGLMixer *mix; + gboolean result = FALSE; + + mix = GST_GL_MIXER (aggregator); + + switch (mode) { + case GST_PAD_MODE_PUSH: + case GST_PAD_MODE_PULL: + result = gst_gl_mixer_activate (mix, active); + break; + default: + result = TRUE; + break; + } + return result; +} + +static gboolean +gst_gl_mixer_query_caps (GstPad * pad, GstAggregator * agg, GstQuery * query) +{ + GstCaps *filter, *caps; + GstStructure *s; + gint n; + + GstVideoAggregator *vagg = GST_VIDEO_AGGREGATOR (agg); + + gst_query_parse_caps (query, &filter); + + if (GST_VIDEO_INFO_FORMAT (&vagg->info) != GST_VIDEO_FORMAT_UNKNOWN) { + caps = gst_video_info_to_caps (&vagg->info); + } else { + caps = gst_pad_get_pad_template_caps (agg->srcpad); + } + + caps = gst_caps_make_writable (caps); + + n = gst_caps_get_size (caps) - 1; + for (; n >= 0; n--) { + s = gst_caps_get_structure (caps, n); + gst_structure_set (s, "width", GST_TYPE_INT_RANGE, 1, G_MAXINT, + "height", GST_TYPE_INT_RANGE, 1, G_MAXINT, NULL); + if (GST_VIDEO_INFO_FPS_D (&vagg->info) != 0) { + gst_structure_set (s, + "framerate", GST_TYPE_FRACTION_RANGE, 0, 1, G_MAXINT, 1, NULL); + } + } + + if (filter) + caps = gst_caps_intersect_full (filter, caps, GST_CAPS_INTERSECT_FIRST); + + gst_query_set_caps_result (query, caps); + gst_caps_unref (caps); + + return TRUE; +} + +static gboolean +gst_gl_mixer_src_query (GstAggregator * agg, GstQuery * query) +{ + gboolean res = FALSE; + GstGLMixer *mix = GST_GL_MIXER (agg); + + switch (GST_QUERY_TYPE (query)) { + case GST_QUERY_CONTEXT: + { + res = gst_gl_handle_context_query ((GstElement *) mix, query, + &mix->display); + break; + } + case GST_QUERY_CAPS: + res = gst_gl_mixer_query_caps (agg->srcpad, agg, query); + break; + default: + res = GST_AGGREGATOR_CLASS (parent_class)->src_query (agg, query); + break; + } + + return res; +} + +static GstFlowReturn +gst_gl_mixer_get_output_buffer (GstVideoAggregator * videoaggregator, + GstBuffer ** outbuf) +{ + GstGLMixer *mix = GST_GL_MIXER (videoaggregator); + + if (!mix->priv->pool_active) { + if (!gst_buffer_pool_set_active (mix->priv->pool, TRUE)) { + GST_ELEMENT_ERROR (mix, RESOURCE, SETTINGS, + ("failed to activate bufferpool"), ("failed to activate bufferpool")); + return GST_FLOW_ERROR; + } + mix->priv->pool_active = TRUE; + } + + return gst_buffer_pool_acquire_buffer (mix->priv->pool, outbuf, NULL); +} + +static gboolean +gst_gl_mixer_decide_allocation (GstGLMixer * mix, GstQuery * query) +{ + GstGLMixerClass *mixer_class = GST_GL_MIXER_GET_CLASS (mix); + GstBufferPool *pool = NULL; + GstStructure *config; + GstCaps *caps; + guint min, max, size; + gboolean update_pool; + GError *error = NULL; + guint idx; + guint out_width, out_height; + GstGLContext *other_context = NULL; + GstVideoAggregator *vagg = GST_VIDEO_AGGREGATOR (mix); + + gst_query_parse_allocation (query, &caps, NULL); + + if (gst_query_get_n_allocation_pools (query) > 0) { + gst_query_parse_nth_allocation_pool (query, 0, &pool, &size, &min, &max); + + update_pool = TRUE; + } else { + GstVideoInfo vinfo; + + gst_video_info_init (&vinfo); + gst_video_info_from_caps (&vinfo, caps); + size = vinfo.size; + min = max = 0; + update_pool = FALSE; + } + + if (!gst_gl_ensure_display (mix, &mix->display)) + return FALSE; + + if (gst_query_find_allocation_meta (query, + GST_VIDEO_GL_TEXTURE_UPLOAD_META_API_TYPE, &idx)) { + GstGLContext *context; + const GstStructure *upload_meta_params; + gpointer handle; + gchar *type; + gchar *apis; + + gst_query_parse_nth_allocation_meta (query, idx, &upload_meta_params); + if (upload_meta_params) { + if (gst_structure_get (upload_meta_params, "gst.gl.GstGLContext", + GST_GL_TYPE_CONTEXT, &context, NULL) && context) { + GstGLContext *old = mix->context; + + mix->context = context; + if (old) + gst_object_unref (old); + } else if (gst_structure_get (upload_meta_params, "gst.gl.context.handle", + G_TYPE_POINTER, &handle, "gst.gl.context.type", G_TYPE_STRING, + &type, "gst.gl.context.apis", G_TYPE_STRING, &apis, NULL) + && handle) { + GstGLPlatform platform = GST_GL_PLATFORM_NONE; + GstGLAPI gl_apis; + + GST_DEBUG ("got GL context handle 0x%p with type %s and apis %s", + handle, type, apis); + + platform = gst_gl_platform_from_string (type); + gl_apis = gst_gl_api_from_string (apis); + + if (gl_apis && platform) + other_context = + gst_gl_context_new_wrapped (mix->display, (guintptr) handle, + platform, gl_apis); + } + } + } + + if (!mix->context) { + mix->context = gst_gl_context_new (mix->display); + if (!gst_gl_context_create (mix->context, other_context, &error)) + goto context_error; + } + + out_width = GST_VIDEO_INFO_WIDTH (&vagg->info); + out_height = GST_VIDEO_INFO_HEIGHT (&vagg->info); + + if (!gst_gl_context_gen_fbo (mix->context, out_width, out_height, + &mix->fbo, &mix->depthbuffer)) + goto context_error; + + if (mix->out_tex_id) + gst_gl_context_del_texture (mix->context, &mix->out_tex_id); + gst_gl_context_gen_texture (mix->context, &mix->out_tex_id, + GST_VIDEO_FORMAT_RGBA, out_width, out_height); + + if (mixer_class->set_caps) + mixer_class->set_caps (mix, caps); + + if (!pool) + pool = gst_gl_buffer_pool_new (mix->context); + + config = gst_buffer_pool_get_config (pool); + gst_buffer_pool_config_set_params (config, caps, size, min, max); + + gst_buffer_pool_config_add_option (config, GST_BUFFER_POOL_OPTION_VIDEO_META); + + gst_buffer_pool_set_config (pool, config); + + if (update_pool) + gst_query_set_nth_allocation_pool (query, 0, pool, size, min, max); + else + gst_query_add_allocation_pool (query, pool, size, min, max); + + gst_object_unref (pool); + + return TRUE; + +context_error: + { + GST_ELEMENT_ERROR (mix, RESOURCE, NOT_FOUND, ("%s", error->message), + (NULL)); + return FALSE; + } +} + +/* takes ownership of the pool, allocator and query */ +static gboolean +gst_gl_mixer_set_allocation (GstGLMixer * mix, + GstBufferPool * pool, GstAllocator * allocator, + GstAllocationParams * params, GstQuery * query) +{ + GstAllocator *oldalloc; + GstBufferPool *oldpool; + GstQuery *oldquery; + GstGLMixerPrivate *priv = mix->priv; + + GST_DEBUG ("storing allocation query"); + + GST_OBJECT_LOCK (mix); + oldpool = priv->pool; + priv->pool = pool; + priv->pool_active = FALSE; + + oldalloc = priv->allocator; + priv->allocator = allocator; + + oldquery = priv->query; + priv->query = query; + + if (params) + priv->params = *params; + else + gst_allocation_params_init (&priv->params); + GST_OBJECT_UNLOCK (mix); + + if (oldpool) { + GST_DEBUG_OBJECT (mix, "deactivating old pool %p", oldpool); + gst_buffer_pool_set_active (oldpool, FALSE); + gst_object_unref (oldpool); + } + if (oldalloc) { + gst_object_unref (oldalloc); + } + if (oldquery) { + gst_query_unref (oldquery); + } + return TRUE; +} + +static gboolean +gst_gl_mixer_do_bufferpool (GstGLMixer * mix, GstCaps * outcaps) +{ + GstQuery *query; + gboolean result = TRUE; + GstBufferPool *pool = NULL; + GstAllocator *allocator; + GstAllocationParams params; + GstAggregator *agg = GST_AGGREGATOR (mix); + + /* find a pool for the negotiated caps now */ + GST_DEBUG_OBJECT (mix, "doing allocation query"); + query = gst_query_new_allocation (outcaps, TRUE); + if (!gst_pad_peer_query (agg->srcpad, query)) { + /* not a problem, just debug a little */ + GST_DEBUG_OBJECT (mix, "peer ALLOCATION query failed"); + } + + GST_DEBUG_OBJECT (mix, "calling decide_allocation"); + result = gst_gl_mixer_decide_allocation (mix, query); + + GST_DEBUG_OBJECT (mix, "ALLOCATION (%d) params: %" GST_PTR_FORMAT, result, + query); + + if (!result) + goto no_decide_allocation; + + /* we got configuration from our peer or the decide_allocation method, + * parse them */ + if (gst_query_get_n_allocation_params (query) > 0) { + gst_query_parse_nth_allocation_param (query, 0, &allocator, ¶ms); + } else { + allocator = NULL; + gst_allocation_params_init (¶ms); + } + + if (gst_query_get_n_allocation_pools (query) > 0) + gst_query_parse_nth_allocation_pool (query, 0, &pool, NULL, NULL, NULL); + + /* now store */ + result = gst_gl_mixer_set_allocation (mix, pool, allocator, ¶ms, query); + + return result; + + /* Errors */ +no_decide_allocation: + { + GST_WARNING_OBJECT (mix, "Failed to decide allocation"); + gst_query_unref (query); + + return result; + } +} + +gboolean +gst_gl_mixer_process_textures (GstGLMixer * mix, GstBuffer * outbuf) +{ + guint i; + GList *walk; + guint out_tex; + gboolean res = TRUE; + guint array_index = 0; + GstVideoFrame out_frame; + gboolean out_gl_wrapped = FALSE; + GstElement *element = GST_ELEMENT (mix); + GstVideoAggregator *vagg = GST_VIDEO_AGGREGATOR (mix); + GstGLMixerClass *mix_class = GST_GL_MIXER_GET_CLASS (mix); + + GST_TRACE ("Processing buffers"); + + if (!gst_video_frame_map (&out_frame, &vagg->info, outbuf, + GST_MAP_WRITE | GST_MAP_GL)) { + return FALSE; + } + + if (gst_is_gl_memory (out_frame.map[0].memory)) { + out_tex = *(guint *) out_frame.data[0]; + } else { + GST_INFO ("Output Buffer does not contain correct memory, " + "attempting to wrap for download"); + + out_tex = mix->out_tex_id;; + + if (!mix->download) + mix->download = gst_gl_download_new (mix->context); + + gst_gl_download_set_format (mix->download, &out_frame.info); + out_gl_wrapped = TRUE; + } + + GST_OBJECT_LOCK (mix); + walk = element->sinkpads; + + i = mix->frames->len; + g_ptr_array_set_size (mix->frames, element->numsinkpads); + for (; i < element->numsinkpads; i++) + mix->frames->pdata[i] = g_slice_new0 (GstGLMixerFrameData); + while (walk) { + GstGLMixerPad *pad = GST_GL_MIXER_PAD (walk->data); + GstVideoAggregatorPad *vaggpad = walk->data; + GstGLMixerFrameData *frame; + + frame = g_ptr_array_index (mix->frames, array_index); + frame->pad = pad; + frame->texture = 0; + + walk = g_list_next (walk); + + if (vaggpad->buffer != NULL) { + guint in_tex; + + if (!pad->upload) { + pad->upload = gst_gl_upload_new (mix->context); + + gst_gl_upload_set_format (pad->upload, &vaggpad->info); + } + + if (!gst_gl_upload_perform_with_buffer (pad->upload, + vaggpad->buffer, &in_tex)) { + ++array_index; + pad->mapped = FALSE; + continue; + } + pad->mapped = TRUE; + + frame->texture = in_tex; + } + ++array_index; + } + + mix_class->process_textures (mix, mix->frames, out_tex); + + if (out_gl_wrapped) { + if (!gst_gl_download_perform_with_data (mix->download, out_tex, + out_frame.data)) { + GST_ELEMENT_ERROR (mix, RESOURCE, NOT_FOUND, ("%s", + "Failed to download video frame"), (NULL)); + res = FALSE; + goto out; + } + } + +out: + i = 0; + walk = GST_ELEMENT (mix)->sinkpads; + while (walk) { + GstGLMixerPad *pad = GST_GL_MIXER_PAD (walk->data); + + if (pad->mapped) + gst_gl_upload_release_buffer (pad->upload); + + pad->mapped = FALSE; + walk = g_list_next (walk); + i++; + } + GST_OBJECT_UNLOCK (mix); + + gst_video_frame_unmap (&out_frame); + + return res; +} + +static gboolean +gst_gl_mixer_process_buffers (GstGLMixer * mix, GstBuffer * outbuf) +{ + GList *walk; + guint i, array_index = 0; + GstElement *element = GST_ELEMENT (mix); + GstGLMixerClass *mix_class = GST_GL_MIXER_GET_CLASS (mix); + + GST_OBJECT_LOCK (mix); + walk = GST_ELEMENT (mix)->sinkpads; + i = mix->frames->len; + g_ptr_array_set_size (mix->frames, element->numsinkpads); + for (; i < element->numsinkpads; i++) + mix->frames->pdata[i] = g_slice_new0 (GstGLMixerFrameData); + while (walk) { /* We walk with this list because it's ordered */ + GstVideoAggregatorPad *vaggpad = walk->data; + + walk = g_list_next (walk); + + if (vaggpad->buffer != NULL) { + /* put buffer into array */ + mix->array_buffers->pdata[array_index] = vaggpad->buffer; + } + ++array_index; + } + GST_OBJECT_UNLOCK (mix); + + return mix_class->process_buffers (mix, mix->array_buffers, outbuf); +} + + + +static GstFlowReturn +gst_gl_mixer_aggregate_frames (GstVideoAggregator * vagg, GstBuffer * outbuf) +{ + gboolean res = FALSE; + GstGLMixer *mix = GST_GL_MIXER (vagg); + GstGLMixerClass *mix_class = GST_GL_MIXER_GET_CLASS (vagg); + + if (mix_class->process_buffers) + res = gst_gl_mixer_process_buffers (mix, outbuf); + else if (mix_class->process_textures) + res = gst_gl_mixer_process_textures (mix, outbuf); + + return res ? GST_FLOW_OK : GST_FLOW_ERROR; +} + +static void +gst_gl_mixer_get_property (GObject * object, + guint prop_id, GValue * value, GParamSpec * pspec) +{ + switch (prop_id) { + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static void +gst_gl_mixer_set_property (GObject * object, + guint prop_id, const GValue * value, GParamSpec * pspec) +{ + switch (prop_id) { + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static gboolean +_clean_upload (GstAggregator * agg, GstPad * aggpad, gpointer udata) +{ + GstGLMixerPad *pad = GST_GL_MIXER_PAD (aggpad); + + if (pad->upload) { + gst_object_unref (pad->upload); + pad->upload = NULL; + } + + return TRUE; +} + +static void +_free_glmixer_frame_data (GstGLMixerFrameData * frame) +{ + g_slice_free1 (sizeof (GstGLMixerFrameData), frame); +} + +static gboolean +gst_gl_mixer_start (GstAggregator * agg) +{ + guint i; + GstGLMixer *mix = GST_GL_MIXER (agg); + GstElement *element = GST_ELEMENT (agg); + + if (!GST_AGGREGATOR_CLASS (parent_class)->start (agg)) + return FALSE; + + GST_OBJECT_LOCK (mix); + mix->array_buffers = g_ptr_array_new_full (element->numsinkpads, + (GDestroyNotify) _free_glmixer_frame_data); + mix->frames = g_ptr_array_new_full (element->numsinkpads, NULL); + + g_ptr_array_set_size (mix->array_buffers, element->numsinkpads); + g_ptr_array_set_size (mix->frames, element->numsinkpads); + + for (i = 0; i < element->numsinkpads; i++) + mix->frames->pdata[i] = g_slice_new0 (GstGLMixerFrameData); + + GST_OBJECT_UNLOCK (mix); + + return TRUE; +} + +static gboolean +gst_gl_mixer_stop (GstAggregator * agg) +{ + GstGLMixer *mix = GST_GL_MIXER (agg); + GstGLMixerClass *mixer_class = GST_GL_MIXER_GET_CLASS (mix); + + if (!GST_AGGREGATOR_CLASS (parent_class)->stop (agg)) + return FALSE; + + GST_OBJECT_LOCK (agg); + g_ptr_array_free (mix->frames, TRUE); + mix->frames = NULL; + g_ptr_array_free (mix->array_buffers, TRUE); + mix->array_buffers = NULL; + GST_OBJECT_UNLOCK (agg); + + if (mixer_class->reset) + mixer_class->reset (mix); + if (mix->fbo) { + gst_gl_context_del_fbo (mix->context, mix->fbo, mix->depthbuffer); + mix->fbo = 0; + mix->depthbuffer = 0; + } + if (mix->download) { + gst_object_unref (mix->download); + mix->download = NULL; + } + + gst_aggregator_iterate_sinkpads (GST_AGGREGATOR (mix), _clean_upload, NULL); + + if (mix->priv->query) { + gst_query_unref (mix->priv->query); + mix->priv->query = NULL; + } + + if (mix->priv->pool) { + gst_object_unref (mix->priv->pool); + mix->priv->pool = NULL; + } + + if (mix->display) { + gst_object_unref (mix->display); + mix->display = NULL; + } + + if (mix->context) { + gst_object_unref (mix->context); + mix->context = NULL; + } + gst_gl_mixer_reset (mix); + + return TRUE; +} diff --git a/ext/gl/gstglmixer.h b/ext/gl/gstglmixer.h new file mode 100644 index 0000000000..a9e3a3c6cb --- /dev/null +++ b/ext/gl/gstglmixer.h @@ -0,0 +1,95 @@ +/* + * GStreamer + * Copyright (C) 2009 Julien Isorce + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifndef __GST_GL_MIXER_H__ +#define __GST_GL_MIXER_H__ + +#include +#include +#include +#include "gstglmixerpad.h" + +G_BEGIN_DECLS + +#define GST_TYPE_GL_MIXER (gst_gl_mixer_get_type()) +#define GST_GL_MIXER(obj) \ + (G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_GL_MIXER, GstGLMixer)) +#define GST_GL_MIXER_CLASS(klass) \ + (G_TYPE_CHECK_CLASS_CAST((klass),GST_TYPE_GL_MIXER, GstGLMixerClass)) +#define GST_IS_GL_MIXER(obj) \ + (G_TYPE_CHECK_INSTANCE_TYPE((obj),GST_TYPE_GL_MIXER)) +#define GST_IS_GL_MIXER_CLASS(klass) \ + (G_TYPE_CHECK_CLASS_TYPE((klass),GST_TYPE_GL_MIXER)) +#define GST_GL_MIXER_GET_CLASS(obj) \ + (G_TYPE_INSTANCE_GET_CLASS((obj),GST_TYPE_GL_MIXER,GstGLMixerClass)) + +typedef struct _GstGLMixer GstGLMixer; +typedef struct _GstGLMixerClass GstGLMixerClass; +typedef struct _GstGLMixerPrivate GstGLMixerPrivate; +typedef struct _GstGLMixerFrameData GstGLMixerFrameData; + +typedef gboolean (*GstGLMixerSetCaps) (GstGLMixer* mixer, + GstCaps* outcaps); +typedef void (*GstGLMixerReset) (GstGLMixer *mixer); +typedef gboolean (*GstGLMixerProcessFunc) (GstGLMixer *mix, + GPtrArray *buffers, GstBuffer *outbuf); +typedef gboolean (*GstGLMixerProcessTextures) (GstGLMixer *mix, + GPtrArray *frames, guint out_tex); + +struct _GstGLMixer +{ + GstVideoAggregator vaggregator; + + GstGLMixerPrivate *priv; + + GPtrArray *array_buffers; + GPtrArray *frames; + + GLuint out_tex_id; + GstGLDownload *download; + + GstGLDisplay *display; + GstGLContext *context; + GLuint fbo; + GLuint depthbuffer; +}; + +struct _GstGLMixerClass +{ + GstVideoAggregatorClass parent_class; + + GstGLMixerSetCaps set_caps; + GstGLMixerReset reset; + GstGLMixerProcessFunc process_buffers; + GstGLMixerProcessTextures process_textures; +}; + +struct _GstGLMixerFrameData +{ + GstGLMixerPad *pad; + guint texture; +}; + +GType gst_gl_mixer_get_type(void); + +gboolean gst_gl_mixer_process_textures (GstGLMixer * mix, GstBuffer * outbuf); + +G_END_DECLS +#endif /* __GST_GL_MIXER_H__ */ diff --git a/ext/gl/gstglmosaic.h b/ext/gl/gstglmosaic.h index 1da9dbe29c..49c99e7754 100644 --- a/ext/gl/gstglmosaic.h +++ b/ext/gl/gstglmosaic.h @@ -21,7 +21,7 @@ #ifndef _GST_GL_MOSAIC_H_ #define _GST_GL_MOSAIC_H_ -#include +#include "gstglmixer.h" G_BEGIN_DECLS diff --git a/ext/gl/gstglvideomixer.h b/ext/gl/gstglvideomixer.h index 69a077100c..ae2f367a89 100644 --- a/ext/gl/gstglvideomixer.h +++ b/ext/gl/gstglvideomixer.h @@ -21,8 +21,8 @@ #ifndef _GST_GL_VIDEO_MIXER_H_ #define _GST_GL_VIDEO_MIXER_H_ -#include -#include +#include "gstglmixer.h" +#include "gstglmixerpad.h" G_BEGIN_DECLS From e9bf900d333e87495a3d6bf2116628c3d1171e23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wang=20Xin-yu=20=28=E7=8E=8B=E6=98=95=E5=AE=87=29?= Date: Wed, 23 Jul 2014 10:25:31 +0800 Subject: [PATCH 029/381] gl: fix multi gl object leaks 1. fix FBO leaks in decide_allocation 2. fix texture leaks in decide_allocation and reset 3. fix texture leaks in FBO incomplete error path --- ext/gl/gstglmixer.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ext/gl/gstglmixer.c b/ext/gl/gstglmixer.c index 626eb967c7..0d4d7e92ba 100644 --- a/ext/gl/gstglmixer.c +++ b/ext/gl/gstglmixer.c @@ -641,6 +641,12 @@ gst_gl_mixer_decide_allocation (GstGLMixer * mix, GstQuery * query) out_width = GST_VIDEO_INFO_WIDTH (&vagg->info); out_height = GST_VIDEO_INFO_HEIGHT (&vagg->info); + if (mix->fbo) { + gst_gl_context_del_fbo (mix->context, mix->fbo, mix->depthbuffer); + mix->fbo = 0; + mix->depthbuffer = 0; + } + if (!gst_gl_context_gen_fbo (mix->context, out_width, out_height, &mix->fbo, &mix->depthbuffer)) goto context_error; From 5cddc44b0a3684fa2c9930381af8463c5335367d Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Thu, 7 Aug 2014 19:18:49 +1000 Subject: [PATCH 030/381] glvideomixer: add a background property That's compatible with the compositor/videomixer property https://bugzilla.gnome.org/show_bug.cgi?id=731954 --- ext/gl/gstglvideomixer.c | 151 ++++++++++++++++++++++++++++++++++++++- ext/gl/gstglvideomixer.h | 21 ++++++ 2 files changed, 170 insertions(+), 2 deletions(-) diff --git a/ext/gl/gstglvideomixer.c b/ext/gl/gstglvideomixer.c index 7dcb9367a6..2564887a0e 100644 --- a/ext/gl/gstglvideomixer.c +++ b/ext/gl/gstglvideomixer.c @@ -47,7 +47,9 @@ GST_DEBUG_CATEGORY_STATIC (GST_CAT_DEFAULT); enum { PROP_0, + PROP_BACKGROUND, }; +#define DEFAULT_BACKGROUND GST_GL_VIDEO_MIXER_BACKGROUND_CHECKER #define DEBUG_INIT \ GST_DEBUG_CATEGORY_INIT (gst_gl_video_mixer_debug, "glvideomixer", 0, "glvideomixer element"); @@ -69,6 +71,7 @@ static gboolean gst_gl_video_mixer_process_textures (GstGLMixer * mixer, GPtrArray * in_frames, guint out_tex); static void gst_gl_video_mixer_callback (gpointer stuff); +/* *INDENT-OFF* */ /* vertex source */ static const gchar *video_mixer_v_src = "attribute vec4 a_position; \n" @@ -93,6 +96,38 @@ static const gchar *video_mixer_f_src = " gl_FragColor = vec4(rgba.rgb, rgba.a * alpha);\n" "} \n"; +/* checker vertex source */ +static const gchar *checker_v_src = + "attribute vec4 a_position;\n" + "void main()\n" + "{\n" + " gl_Position = a_position;\n" + "}\n"; + +/* checker fragment source */ +static const gchar *checker_f_src = + "#ifdef GL_ES\n" + "precision mediump float;\n" + "#endif\n" + "const float blocksize = 8.0;\n" + "void main ()\n" + "{\n" + " vec4 high = vec4(0.667, 0.667, 0.667, 1.0);\n" + " vec4 low = vec4(0.333, 0.333, 0.333, 1.0);\n" + " if (mod(gl_FragCoord.x, blocksize * 2.0) >= blocksize) {\n" + " if (mod(gl_FragCoord.y, blocksize * 2.0) >= blocksize)\n" + " gl_FragColor = low;\n" + " else\n" + " gl_FragColor = high;\n" + " } else {\n" + " if (mod(gl_FragCoord.y, blocksize * 2.0) < blocksize)\n" + " gl_FragColor = low;\n" + " else\n" + " gl_FragColor = high;\n" + " }\n" + "}\n"; +/* *INDENT-ON* */ + #define GST_TYPE_GL_VIDEO_MIXER_PAD (gst_gl_video_mixer_pad_get_type()) #define GST_GL_VIDEO_MIXER_PAD(obj) \ (G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_GL_VIDEO_MIXER_PAD, GstGLVideoMixerPad)) @@ -247,6 +282,28 @@ gst_gl_video_mixer_pad_set_property (GObject * object, guint prop_id, gst_object_unref (mix); } +#define GST_GL_TYPE_VIDEO_MIXER_BACKGROUND (gst_gl_video_mixer_background_get_type()) +static GType +gst_gl_video_mixer_background_get_type (void) +{ + static GType mixer_background_type = 0; + + static const GEnumValue mixer_background[] = { + {GST_GL_VIDEO_MIXER_BACKGROUND_CHECKER, "Checker pattern", "checker"}, + {GST_GL_VIDEO_MIXER_BACKGROUND_BLACK, "Black", "black"}, + {GST_GL_VIDEO_MIXER_BACKGROUND_WHITE, "White", "white"}, + {GST_GL_VIDEO_MIXER_BACKGROUND_TRANSPARENT, + "Transparent Background to enable further compositing", "transparent"}, + {0, NULL, NULL}, + }; + + if (!mixer_background_type) { + mixer_background_type = + g_enum_register_static ("GstGLVideoMixerBackground", mixer_background); + } + return mixer_background_type; +} + static void gst_gl_video_mixer_class_init (GstGLVideoMixerClass * klass) { @@ -265,6 +322,11 @@ gst_gl_video_mixer_class_init (GstGLVideoMixerClass * klass) "Filter/Effect/Video/Compositor", "OpenGL video_mixer", "Julien Isorce "); + g_object_class_install_property (gobject_class, PROP_BACKGROUND, + g_param_spec_enum ("background", "Background", "Background type", + GST_GL_TYPE_VIDEO_MIXER_BACKGROUND, + DEFAULT_BACKGROUND, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + GST_GL_MIXER_CLASS (klass)->set_caps = gst_gl_video_mixer_init_shader; GST_GL_MIXER_CLASS (klass)->reset = gst_gl_video_mixer_reset; GST_GL_MIXER_CLASS (klass)->process_textures = @@ -278,6 +340,7 @@ gst_gl_video_mixer_class_init (GstGLVideoMixerClass * klass) static void gst_gl_video_mixer_init (GstGLVideoMixer * video_mixer) { + video_mixer->background = DEFAULT_BACKGROUND; video_mixer->shader = NULL; video_mixer->input_frames = NULL; } @@ -286,7 +349,12 @@ static void gst_gl_video_mixer_set_property (GObject * object, guint prop_id, const GValue * value, GParamSpec * pspec) { + GstGLVideoMixer *mixer = GST_GL_VIDEO_MIXER (object); + switch (prop_id) { + case PROP_BACKGROUND: + mixer->background = g_value_get_enum (value); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); break; @@ -297,7 +365,12 @@ static void gst_gl_video_mixer_get_property (GObject * object, guint prop_id, GValue * value, GParamSpec * pspec) { + GstGLVideoMixer *mixer = GST_GL_VIDEO_MIXER (object); + switch (prop_id) { + case PROP_BACKGROUND: + g_value_set_enum (value, mixer->background); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); break; @@ -360,6 +433,10 @@ gst_gl_video_mixer_reset (GstGLMixer * mixer) if (video_mixer->shader) gst_gl_context_del_shader (mixer->context, video_mixer->shader); video_mixer->shader = NULL; + + if (video_mixer->checker) + gst_gl_context_del_shader (mixer->context, video_mixer->checker); + video_mixer->checker = NULL; } static gboolean @@ -391,6 +468,76 @@ gst_gl_video_mixer_process_textures (GstGLMixer * mix, GPtrArray * frames, return TRUE; } +static gboolean +_draw_checker_background (GstGLVideoMixer * video_mixer) +{ + GstGLMixer *mixer = GST_GL_MIXER (video_mixer); + const GstGLFuncs *gl = mixer->context->gl_vtable; + gint attr_position_loc = 0; + + const GLushort indices[] = { + 0, 1, 2, + 0, 2, 3 + }; + /* *INDENT-OFF* */ + gfloat v_vertices[] = { + -1.0,-1.0,-1.0f, + 1.0,-1.0,-1.0f, + 1.0, 1.0,-1.0f, + -1.0, 1.0,-1.0f, + }; + /* *INDENT-ON* */ + + attr_position_loc = + gst_gl_shader_get_attribute_location (video_mixer->shader, "a_position"); + + if (!video_mixer->checker) { + if (!gst_gl_context_gen_shader (mixer->context, checker_v_src, + checker_f_src, &video_mixer->checker)) + return FALSE; + } + + gst_gl_shader_use (video_mixer->checker); + + gl->VertexAttribPointer (attr_position_loc, 3, GL_FLOAT, + GL_FALSE, 3 * sizeof (GLfloat), &v_vertices[0]); + + gl->EnableVertexAttribArray (attr_position_loc); + + gl->DrawElements (GL_TRIANGLES, 6, GL_UNSIGNED_SHORT, indices); + + return TRUE; +} + +static gboolean +_draw_background (GstGLVideoMixer * video_mixer) +{ + GstGLMixer *mixer = GST_GL_MIXER (video_mixer); + const GstGLFuncs *gl = mixer->context->gl_vtable; + + switch (video_mixer->background) { + case GST_GL_VIDEO_MIXER_BACKGROUND_BLACK: + gl->ClearColor (0.0, 0.0, 0.0, 1.0); + gl->Clear (GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT); + break; + case GST_GL_VIDEO_MIXER_BACKGROUND_WHITE: + gl->ClearColor (1.0, 1.0, 1.0, 1.0); + gl->Clear (GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT); + break; + case GST_GL_VIDEO_MIXER_BACKGROUND_TRANSPARENT: + gl->ClearColor (0.0, 0.0, 0.0, 0.0); + gl->Clear (GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT); + break; + case GST_GL_VIDEO_MIXER_BACKGROUND_CHECKER: + return _draw_checker_background (video_mixer); + break; + default: + break; + } + + return TRUE; +} + /* opengl scene, params: input texture (not the output mixer->texture) */ static void gst_gl_video_mixer_callback (gpointer stuff) @@ -420,8 +567,8 @@ gst_gl_video_mixer_callback (gpointer stuff) gl->Disable (GL_DEPTH_TEST); gl->Disable (GL_CULL_FACE); - gl->ClearColor (0.0, 0.0, 0.0, 0.0); - gl->Clear (GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT); + if (!_draw_background (video_mixer)) + return; gst_gl_shader_use (video_mixer->shader); diff --git a/ext/gl/gstglvideomixer.h b/ext/gl/gstglvideomixer.h index ae2f367a89..d7dd070737 100644 --- a/ext/gl/gstglvideomixer.h +++ b/ext/gl/gstglvideomixer.h @@ -36,11 +36,32 @@ G_BEGIN_DECLS typedef struct _GstGLVideoMixer GstGLVideoMixer; typedef struct _GstGLVideoMixerClass GstGLVideoMixerClass; +/** + * GstGLVideoMixerBackground: + * @GST_GL_VIDEO_MIXER_BACKGROUND_CHECKER: checker pattern background + * @GST_GL_VIDEO_MIXER_BACKGROUND_BLACK: solid color black background + * @GST_GL_VIDEO_MIXER_BACKGROUND_WHITE: solid color white background + * @GST_GL_VIDEO_MIXER_BACKGROUND_TRANSPARENT: background is left transparent and layers are composited using "A OVER B" composition rules. This is only applicable to AYUV and ARGB (and variants) as it preserves the alpha channel and allows for further mixing. + * + * The different backgrounds compositor can blend over. + */ +typedef enum +{ + GST_GL_VIDEO_MIXER_BACKGROUND_CHECKER, + GST_GL_VIDEO_MIXER_BACKGROUND_BLACK, + GST_GL_VIDEO_MIXER_BACKGROUND_WHITE, + GST_GL_VIDEO_MIXER_BACKGROUND_TRANSPARENT, +} +GstGLVideoMixerBackground; + struct _GstGLVideoMixer { GstGLMixer mixer; + GstGLVideoMixerBackground background; + GstGLShader *shader; + GstGLShader *checker; GPtrArray *input_frames; }; From bc15905c45d62e202f99d21309e887d775bc2584 Mon Sep 17 00:00:00 2001 From: Sebastian Rasmussen Date: Sat, 9 Aug 2014 11:17:44 +0200 Subject: [PATCH 031/381] videoaggregator: Unref allowed caps after usage Fixes https://bugzilla.gnome.org/show_bug.cgi?id=734522 --- gst-libs/gst/video/gstvideoaggregator.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index 09dcbb71dd..02e485cb9d 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -386,6 +386,8 @@ gst_videoaggregator_update_converters (GstVideoAggregator * vagg) if (!downstream_caps || gst_caps_is_empty (downstream_caps)) { GST_INFO_OBJECT (vagg, "No downstream caps found %" GST_PTR_FORMAT, downstream_caps); + if (downstream_caps) + gst_caps_unref (downstream_caps); return FALSE; } From 0ab1136f4e04626aec8b676c58650d1f393d13e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim-Philipp=20M=C3=BCller?= Date: Sun, 10 Aug 2014 18:07:28 +0100 Subject: [PATCH 032/381] opengl: update element docs for 1.x --- ext/gl/gstglmosaic.c | 7 ++++++- ext/gl/gstglvideomixer.c | 8 +++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/ext/gl/gstglmosaic.c b/ext/gl/gstglmosaic.c index 5a5c41835e..1274a3dcec 100644 --- a/ext/gl/gstglmosaic.c +++ b/ext/gl/gstglmosaic.c @@ -29,7 +29,12 @@ * * Examples * |[ - * gst-launch-0.10 videotestsrc ! "video/x-raw-yuv, format=(fourcc)YUY2" ! glupload ! queue ! glmosaic name=m ! glimagesink videotestsrc pattern=12 ! "video/x-raw-yuv, format=(fourcc)I420, framerate=(fraction)5/1, width=100, height=200" ! glupload ! queue ! m. videotestsrc ! "video/x-raw-rgb, framerate=(fraction)15/1, width=1500, height=1500" ! glupload ! gleffects effect=3 ! queue ! m. videotestsrc ! glupload ! gleffects effect=2 ! queue ! m. videotestsrc ! glupload ! glfiltercube ! queue ! m. videotestsrc ! glupload ! gleffects effect=6 ! queue ! m. + * gst-launch-1.0 videotestsrc ! video/x-raw, format=YUY2 ! queue ! glmosaic name=m ! glimagesink \ + * videotestsrc pattern=12 ! video/x-raw, format=I420, framerate=5/1, width=100, height=200 ! queue ! m. \ + * videotestsrc ! video/x-raw, framerate=15/1, width=1500, height=1500 ! gleffects effect=3 ! queue ! m. \ + * videotestsrc ! gleffects effect=2 ! queue ! m. \ + * videotestsrc ! glfiltercube ! queue ! m. \ + * videotestsrc ! gleffects effect=6 ! queue ! m. * ]| * FBO (Frame Buffer Object) is required. * diff --git a/ext/gl/gstglvideomixer.c b/ext/gl/gstglvideomixer.c index 2564887a0e..18b7b208ee 100644 --- a/ext/gl/gstglvideomixer.c +++ b/ext/gl/gstglvideomixer.c @@ -29,7 +29,13 @@ * * Examples * |[ - * gst-launch-0.10 videotestsrc ! "video/x-raw-yuv, format=(fourcc)YUY2" ! glupload ! queue ! glvideomixer name=m ! glimagesink videotestsrc pattern=12 ! "video/x-raw-yuv, format=(fourcc)I420, framerate=(fraction)5/1, width=100, height=200" ! glupload ! queue ! m. videotestsrc ! "video/x-raw-rgb, framerate=(fraction)15/1, width=1500, height=1500" ! glupload ! gleffects effect=3 ! queue ! m. videotestsrc ! glupload ! gleffects effect=2 ! queue ! m. videotestsrc ! glupload ! glfiltercube ! queue ! m. videotestsrc ! glupload ! gleffects effect=6 ! queue ! m. + * gst-launch-1.0 glvideomixer name=m ! glimagesink \ + * videotestsrc ! video/x-raw, format=YUY2 ! m. \ + * videotestsrc pattern=12 ! video/x-raw, format=I420, framerate=5/1, width=100, height=200 ! queue ! m. \ + * videotestsrc ! video/x-raw, format=RGB, framerate=15/1, width=1500, height=1500 ! gleffects effect=3 ! queue ! m. \ + * videotestsrc ! gleffects effect=2 ! queue ! m. \ + * videotestsrc ! glfiltercube ! queue ! m. \ + * videotestsrc ! gleffects effect=6 ! queue ! m. * ]| * FBO (Frame Buffer Object) is required. * From 4b1bf9b52a04579758e31f1c7d6f501963aa3ed8 Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Thu, 7 Aug 2014 19:54:36 +1000 Subject: [PATCH 033/381] videoaggregator: push the caps event as soon as we receive it Along with the required mandatory dependent events. Some elements need to perform an allocation query inside ::negotiated_caps(). Without the caps event being sent prior, downstream elements will be unable to answer and will return an error. https://bugzilla.gnome.org/show_bug.cgi?id=732662 --- gst-libs/gst/video/gstvideoaggregator.c | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index 02e485cb9d..58bcb2ecbc 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -283,7 +283,6 @@ struct _GstVideoAggregatorPrivate /* current caps */ GstCaps *current_caps; - gboolean send_caps; }; G_DEFINE_ABSTRACT_TYPE_WITH_CODE (GstVideoAggregator, gst_videoaggregator, @@ -502,7 +501,7 @@ gst_videoaggregator_src_setcaps (GstVideoAggregator * vagg, GstCaps * caps) if (vagg->priv->current_caps == NULL || gst_caps_is_equal (caps, vagg->priv->current_caps) == FALSE) { gst_caps_replace (&vagg->priv->current_caps, caps); - vagg->priv->send_caps = TRUE; + gst_aggregator_set_src_caps (agg, caps); } done: @@ -1188,11 +1187,6 @@ gst_videoaggregator_aggregate (GstAggregator * agg) if (gst_pad_check_reconfigure (agg->srcpad)) gst_videoaggregator_update_src_caps (vagg); - if (vagg->priv->send_caps) { - gst_aggregator_set_src_caps (agg, vagg->priv->current_caps); - vagg->priv->send_caps = FALSE; - } - GST_VIDEO_AGGREGATOR_LOCK (vagg); if (agg->segment.position == -1) @@ -1652,7 +1646,6 @@ gst_videoaggregator_start (GstAggregator * agg) if (!GST_AGGREGATOR_CLASS (gst_videoaggregator_parent_class)->start (agg)) return FALSE; - vagg->priv->send_caps = TRUE; gst_segment_init (&agg->segment, GST_FORMAT_TIME); gst_caps_replace (&vagg->priv->current_caps, NULL); From 3fe5412357961ade288ca3f866997391e30d6004 Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Tue, 19 Aug 2014 14:43:42 +1000 Subject: [PATCH 034/381] glvideomixer: get the attribute from the correct shader --- ext/gl/gstglvideomixer.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/ext/gl/gstglvideomixer.c b/ext/gl/gstglvideomixer.c index 18b7b208ee..b62cfd727d 100644 --- a/ext/gl/gstglvideomixer.c +++ b/ext/gl/gstglvideomixer.c @@ -494,9 +494,6 @@ _draw_checker_background (GstGLVideoMixer * video_mixer) }; /* *INDENT-ON* */ - attr_position_loc = - gst_gl_shader_get_attribute_location (video_mixer->shader, "a_position"); - if (!video_mixer->checker) { if (!gst_gl_context_gen_shader (mixer->context, checker_v_src, checker_f_src, &video_mixer->checker)) @@ -504,6 +501,8 @@ _draw_checker_background (GstGLVideoMixer * video_mixer) } gst_gl_shader_use (video_mixer->checker); + attr_position_loc = + gst_gl_shader_get_attribute_location (video_mixer->checker, "a_position"); gl->VertexAttribPointer (attr_position_loc, 3, GL_FLOAT, GL_FALSE, 3 * sizeof (GLfloat), &v_vertices[0]); From a555c32f3050dd6d9270a3206b645276bfeada74 Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Tue, 19 Aug 2014 14:44:29 +1000 Subject: [PATCH 035/381] glvideomixer: don't clobber unnecessary GstVideoInfo fields otherwise we might clobber other important fields such as the frame rate. --- ext/gl/gstglvideomixer.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ext/gl/gstglvideomixer.c b/ext/gl/gstglvideomixer.c index b62cfd727d..2b96bc5cae 100644 --- a/ext/gl/gstglvideomixer.c +++ b/ext/gl/gstglvideomixer.c @@ -421,8 +421,8 @@ _update_info (GstVideoAggregator * vagg, GstVideoInfo * info) GST_OBJECT_UNLOCK (vagg); if (best_width > 0 && best_height > 0) { - gst_video_info_set_format (info, GST_VIDEO_INFO_FORMAT (info), - best_width, best_height); + info->width = best_width; + info->height = best_height; ret = TRUE; } From 51b23bbff740d185c76b664697a680a81a1bad52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wang=20Xin-yu=20=28=E7=8E=8B=E6=98=95=E5=AE=87=29?= Date: Thu, 14 Aug 2014 23:51:21 -0400 Subject: [PATCH 036/381] glvideomixer: avoid gl resource race condition between different thread https://bugzilla.gnome.org/show_bug.cgi?id=734830 --- ext/gl/gstglmixer.c | 37 ++++++++++++++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/ext/gl/gstglmixer.c b/ext/gl/gstglmixer.c index 0d4d7e92ba..1420b836c9 100644 --- a/ext/gl/gstglmixer.c +++ b/ext/gl/gstglmixer.c @@ -70,6 +70,10 @@ struct _GstGLMixerPrivate GstAllocator *allocator; GstAllocationParams params; GstQuery *query; + + gboolean gl_resource_ready; + GMutex gl_resource_lock; + GCond gl_resource_cond; }; G_DEFINE_TYPE (GstGLMixerPad, gst_gl_mixer_pad, GST_TYPE_VIDEO_AGGREGATOR_PAD); @@ -427,6 +431,9 @@ gst_gl_mixer_init (GstGLMixer * mix) mix->fbo = 0; mix->depthbuffer = 0; + mix->priv->gl_resource_ready = FALSE; + g_mutex_init (&mix->priv->gl_resource_lock); + g_cond_init (&mix->priv->gl_resource_cond); /* initialize variables */ gst_gl_mixer_reset (mix); } @@ -434,6 +441,10 @@ gst_gl_mixer_init (GstGLMixer * mix) static void gst_gl_mixer_finalize (GObject * object) { + GstGLMixerPrivate *priv = GST_GL_MIXER (object)->priv; + + g_mutex_clear (&priv->gl_resource_lock); + g_cond_clear (&priv->gl_resource_cond); G_OBJECT_CLASS (parent_class)->finalize (object); } @@ -641,6 +652,8 @@ gst_gl_mixer_decide_allocation (GstGLMixer * mix, GstQuery * query) out_width = GST_VIDEO_INFO_WIDTH (&vagg->info); out_height = GST_VIDEO_INFO_HEIGHT (&vagg->info); + g_mutex_lock (&mix->priv->gl_resource_lock); + mix->priv->gl_resource_ready = FALSE; if (mix->fbo) { gst_gl_context_del_fbo (mix->context, mix->fbo, mix->depthbuffer); mix->fbo = 0; @@ -648,8 +661,11 @@ gst_gl_mixer_decide_allocation (GstGLMixer * mix, GstQuery * query) } if (!gst_gl_context_gen_fbo (mix->context, out_width, out_height, - &mix->fbo, &mix->depthbuffer)) + &mix->fbo, &mix->depthbuffer)) { + g_cond_signal (&mix->priv->gl_resource_cond); + g_mutex_unlock (&mix->priv->gl_resource_lock); goto context_error; + } if (mix->out_tex_id) gst_gl_context_del_texture (mix->context, &mix->out_tex_id); @@ -659,6 +675,10 @@ gst_gl_mixer_decide_allocation (GstGLMixer * mix, GstQuery * query) if (mixer_class->set_caps) mixer_class->set_caps (mix, caps); + mix->priv->gl_resource_ready = TRUE; + g_cond_signal (&mix->priv->gl_resource_cond); + g_mutex_unlock (&mix->priv->gl_resource_lock); + if (!pool) pool = gst_gl_buffer_pool_new (mix->context); @@ -797,6 +817,7 @@ gst_gl_mixer_process_textures (GstGLMixer * mix, GstBuffer * outbuf) GstElement *element = GST_ELEMENT (mix); GstVideoAggregator *vagg = GST_VIDEO_AGGREGATOR (mix); GstGLMixerClass *mix_class = GST_GL_MIXER_GET_CLASS (mix); + GstGLMixerPrivate *priv = mix->priv; GST_TRACE ("Processing buffers"); @@ -860,8 +881,22 @@ gst_gl_mixer_process_textures (GstGLMixer * mix, GstBuffer * outbuf) ++array_index; } + g_mutex_lock (&priv->gl_resource_lock); + if (!priv->gl_resource_ready) + g_cond_wait (&priv->gl_resource_cond, &priv->gl_resource_lock); + + if (!priv->gl_resource_ready) { + g_mutex_unlock (&priv->gl_resource_lock); + GST_ERROR_OBJECT (mix, + "fbo used to render can't be created, do not run process_textures"); + res = FALSE; + goto out; + } + mix_class->process_textures (mix, mix->frames, out_tex); + g_mutex_unlock (&priv->gl_resource_lock); + if (out_gl_wrapped) { if (!gst_gl_download_perform_with_data (mix->download, out_tex, out_frame.data)) { From 7a0155e73550aa873a8b5fc6bc8d8249da71dfa5 Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Tue, 19 Aug 2014 17:01:36 +1000 Subject: [PATCH 037/381] glmixer: unref the GstGLUpload in the pad if freed while running Dynamic pipelines that get and release the sink pads will finalize the pad without going through gst_gl_mixer_stop() which is where the upload object is usually freed. Don't leak objects in such case. --- ext/gl/gstglmixer.c | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/ext/gl/gstglmixer.c b/ext/gl/gstglmixer.c index 1420b836c9..62bb4d455d 100644 --- a/ext/gl/gstglmixer.c +++ b/ext/gl/gstglmixer.c @@ -49,6 +49,7 @@ static void gst_gl_mixer_pad_get_property (GObject * object, guint prop_id, GValue * value, GParamSpec * pspec); static void gst_gl_mixer_pad_set_property (GObject * object, guint prop_id, const GValue * value, GParamSpec * pspec); +static void gst_gl_mixer_pad_finalize (GObject * object); static void gst_gl_mixer_set_context (GstElement * element, GstContext * context); @@ -85,6 +86,19 @@ gst_gl_mixer_pad_class_init (GstGLMixerPadClass * klass) gobject_class->set_property = gst_gl_mixer_pad_set_property; gobject_class->get_property = gst_gl_mixer_pad_get_property; + + gobject_class->finalize = gst_gl_mixer_pad_finalize; +} + +static void +gst_gl_mixer_pad_finalize (GObject * object) +{ + GstGLMixerPad *pad = GST_GL_MIXER_PAD (object); + + if (pad->upload) { + gst_object_unref (pad->upload); + pad->upload = NULL; + } } static void From 0dbce9c6abb84ce703d58ea49ced1158d67d5462 Mon Sep 17 00:00:00 2001 From: Ognyan Tonchev Date: Tue, 16 Sep 2014 11:41:16 +0200 Subject: [PATCH 038/381] glmixer: do not leak pool in error cases https://bugzilla.gnome.org/show_bug.cgi?id=736729 --- ext/gl/gstglmixer.c | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/ext/gl/gstglmixer.c b/ext/gl/gstglmixer.c index 62bb4d455d..5aa1beab90 100644 --- a/ext/gl/gstglmixer.c +++ b/ext/gl/gstglmixer.c @@ -600,22 +600,6 @@ gst_gl_mixer_decide_allocation (GstGLMixer * mix, GstQuery * query) GstGLContext *other_context = NULL; GstVideoAggregator *vagg = GST_VIDEO_AGGREGATOR (mix); - gst_query_parse_allocation (query, &caps, NULL); - - if (gst_query_get_n_allocation_pools (query) > 0) { - gst_query_parse_nth_allocation_pool (query, 0, &pool, &size, &min, &max); - - update_pool = TRUE; - } else { - GstVideoInfo vinfo; - - gst_video_info_init (&vinfo); - gst_video_info_from_caps (&vinfo, caps); - size = vinfo.size; - min = max = 0; - update_pool = FALSE; - } - if (!gst_gl_ensure_display (mix, &mix->display)) return FALSE; @@ -686,6 +670,8 @@ gst_gl_mixer_decide_allocation (GstGLMixer * mix, GstQuery * query) gst_gl_context_gen_texture (mix->context, &mix->out_tex_id, GST_VIDEO_FORMAT_RGBA, out_width, out_height); + gst_query_parse_allocation (query, &caps, NULL); + if (mixer_class->set_caps) mixer_class->set_caps (mix, caps); @@ -693,6 +679,20 @@ gst_gl_mixer_decide_allocation (GstGLMixer * mix, GstQuery * query) g_cond_signal (&mix->priv->gl_resource_cond); g_mutex_unlock (&mix->priv->gl_resource_lock); + if (gst_query_get_n_allocation_pools (query) > 0) { + gst_query_parse_nth_allocation_pool (query, 0, &pool, &size, &min, &max); + + update_pool = TRUE; + } else { + GstVideoInfo vinfo; + + gst_video_info_init (&vinfo); + gst_video_info_from_caps (&vinfo, caps); + size = vinfo.size; + min = max = 0; + update_pool = FALSE; + } + if (!pool) pool = gst_gl_buffer_pool_new (mix->context); From a45a5f010108e7d3ae954bd5e64aff29edf0f411 Mon Sep 17 00:00:00 2001 From: Sanjay NM Date: Fri, 19 Sep 2014 15:32:33 +0530 Subject: [PATCH 039/381] gl: Removed unreachable break, unused variable https://bugzilla.gnome.org/show_bug.cgi?id=736957 --- ext/gl/gstglmixer.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/ext/gl/gstglmixer.c b/ext/gl/gstglmixer.c index 5aa1beab90..72291a8bad 100644 --- a/ext/gl/gstglmixer.c +++ b/ext/gl/gstglmixer.c @@ -272,10 +272,9 @@ gst_gl_mixer_sink_query (GstAggregator * agg, GstAggregatorPad * bpad, case GST_QUERY_ALLOCATION: { GstQuery *decide_query = NULL; - gboolean negotiated; GST_OBJECT_LOCK (mix); - if (G_UNLIKELY (!(negotiated = mix->priv->negotiated))) { + if (G_UNLIKELY (!mix->priv->negotiated)) { GST_DEBUG_OBJECT (mix, "not negotiated yet, can't answer ALLOCATION query"); GST_OBJECT_UNLOCK (mix); @@ -624,7 +623,7 @@ gst_gl_mixer_decide_allocation (GstGLMixer * mix, GstQuery * query) G_TYPE_POINTER, &handle, "gst.gl.context.type", G_TYPE_STRING, &type, "gst.gl.context.apis", G_TYPE_STRING, &apis, NULL) && handle) { - GstGLPlatform platform = GST_GL_PLATFORM_NONE; + GstGLPlatform platform; GstGLAPI gl_apis; GST_DEBUG ("got GL context handle 0x%p with type %s and apis %s", From 03c21f78b6ef4a5e0bf2892d0500dfc44ad66970 Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Sun, 21 Sep 2014 21:36:49 +1000 Subject: [PATCH 040/381] glupload: provide the output buffer that is rendered into Allows callers to properly reference count the buffers used for rendering. Fixes a redraw race in glimagesink where the previous buffer (the one used for redraw operations) is freed as soon as the next buffer is uploaded. 1. glimagesink uploads in _prepare() to texture n 1.1 glupload holds buffer n 2. glimagesink _render()s texture n 3. glimagesink uploads texture n+1 3.1 glupload free previous buffer which deletes texture n 3.2 glupload holds buffer n+1 4. glwindow resize/expose 5. glimagesink redraws with texture n The race is that the buffer n (the one used for redrawing) is freed as soon as the buffer n+1 arrives. There could be any amount of time and number of redraws between this event and when buffer n+1 is actually rendered and thus replaces buffer n as the redraw source. https://bugzilla.gnome.org/show_bug.cgi?id=736740 --- ext/gl/gstglmixer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/gl/gstglmixer.c b/ext/gl/gstglmixer.c index 72291a8bad..3fcd728f17 100644 --- a/ext/gl/gstglmixer.c +++ b/ext/gl/gstglmixer.c @@ -882,7 +882,7 @@ gst_gl_mixer_process_textures (GstGLMixer * mix, GstBuffer * outbuf) } if (!gst_gl_upload_perform_with_buffer (pad->upload, - vaggpad->buffer, &in_tex)) { + vaggpad->buffer, &in_tex, NULL)) { ++array_index; pad->mapped = FALSE; continue; From e0459f1b65c78326b7a685c7d74baddd3c19845b Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Wed, 24 Sep 2014 13:13:19 +1000 Subject: [PATCH 041/381] gl: download whenever we have sysmem capsfeatures Otherwise we could pass on a RGBA formatted buffer and downstream would misinterpret that as some other video format. Fixes pipelines of the form gleffects ! tee ! xvimagesink --- ext/gl/gstglmixer.c | 23 +++++++++++++++-------- ext/gl/gstglmixer.h | 2 ++ 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/ext/gl/gstglmixer.c b/ext/gl/gstglmixer.c index 3fcd728f17..04c951fdb5 100644 --- a/ext/gl/gstglmixer.c +++ b/ext/gl/gstglmixer.c @@ -131,6 +131,8 @@ _negotiated_caps (GstVideoAggregator * vagg, GstCaps * caps) mix->priv->negotiated = ret; + gst_caps_replace (&mix->out_caps, caps); + return ret; } @@ -826,32 +828,37 @@ gst_gl_mixer_process_textures (GstGLMixer * mix, GstBuffer * outbuf) gboolean res = TRUE; guint array_index = 0; GstVideoFrame out_frame; - gboolean out_gl_wrapped = FALSE; GstElement *element = GST_ELEMENT (mix); GstVideoAggregator *vagg = GST_VIDEO_AGGREGATOR (mix); GstGLMixerClass *mix_class = GST_GL_MIXER_GET_CLASS (mix); GstGLMixerPrivate *priv = mix->priv; + gboolean to_download = + gst_caps_features_is_equal (GST_CAPS_FEATURES_MEMORY_SYSTEM_MEMORY, + gst_caps_get_features (mix->out_caps, 0)); + GstMapFlags out_map_flags = GST_MAP_WRITE; GST_TRACE ("Processing buffers"); - if (!gst_video_frame_map (&out_frame, &vagg->info, outbuf, - GST_MAP_WRITE | GST_MAP_GL)) { + to_download |= !gst_is_gl_memory (gst_buffer_peek_memory (outbuf, 0)); + + if (!to_download) + out_map_flags |= GST_MAP_GL; + + if (!gst_video_frame_map (&out_frame, &vagg->info, outbuf, out_map_flags)) { return FALSE; } - if (gst_is_gl_memory (out_frame.map[0].memory)) { + if (!to_download) { out_tex = *(guint *) out_frame.data[0]; } else { GST_INFO ("Output Buffer does not contain correct memory, " "attempting to wrap for download"); - out_tex = mix->out_tex_id;; - if (!mix->download) mix->download = gst_gl_download_new (mix->context); gst_gl_download_set_format (mix->download, &out_frame.info); - out_gl_wrapped = TRUE; + out_tex = mix->out_tex_id; } GST_OBJECT_LOCK (mix); @@ -910,7 +917,7 @@ gst_gl_mixer_process_textures (GstGLMixer * mix, GstBuffer * outbuf) g_mutex_unlock (&priv->gl_resource_lock); - if (out_gl_wrapped) { + if (to_download) { if (!gst_gl_download_perform_with_data (mix->download, out_tex, out_frame.data)) { GST_ELEMENT_ERROR (mix, RESOURCE, NOT_FOUND, ("%s", diff --git a/ext/gl/gstglmixer.h b/ext/gl/gstglmixer.h index a9e3a3c6cb..d66ef30c18 100644 --- a/ext/gl/gstglmixer.h +++ b/ext/gl/gstglmixer.h @@ -69,6 +69,8 @@ struct _GstGLMixer GstGLContext *context; GLuint fbo; GLuint depthbuffer; + + GstCaps *out_caps; }; struct _GstGLMixerClass From e0476bf0146d2991e321cfa4e0a503c271f0d656 Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Wed, 24 Sep 2014 14:29:37 +1000 Subject: [PATCH 042/381] glvideomixer: skip input frames with an alpha of 0 --- ext/gl/gstglvideomixer.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/ext/gl/gstglvideomixer.c b/ext/gl/gstglvideomixer.c index 2b96bc5cae..9ffdcb6ad4 100644 --- a/ext/gl/gstglvideomixer.c +++ b/ext/gl/gstglvideomixer.c @@ -611,12 +611,14 @@ gst_gl_video_mixer_callback (gpointer stuff) in_width = GST_VIDEO_INFO_WIDTH (&GST_VIDEO_AGGREGATOR_PAD (pad)->info); in_height = GST_VIDEO_INFO_HEIGHT (&GST_VIDEO_AGGREGATOR_PAD (pad)->info); - if (!frame->texture || in_width <= 0 || in_height <= 0) { - GST_DEBUG ("skipping texture:%u frame:%p width:%u height %u", - frame->texture, frame, in_width, in_height); + if (!frame->texture || in_width <= 0 || in_height <= 0 + || pad->alpha == 0.0f) { + GST_DEBUG ("skipping texture:%u frame:%p width:%u height:%u alpha:%f", + frame->texture, frame, in_width, in_height, pad->alpha); count++; continue; } + in_tex = frame->texture; pad_width = pad->width <= 0 ? in_width : pad->width; pad_height = pad->height <= 0 ? in_height : pad->height; From e4dd599a55c6e9b17cc468e9d61ae8ae60999c07 Mon Sep 17 00:00:00 2001 From: Wim Taymans Date: Wed, 24 Sep 2014 16:55:49 +0200 Subject: [PATCH 043/381] video: use video lib conversion code instead of copy --- gst-libs/gst/video/gstvideoaggregator.c | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index 58bcb2ecbc..a09ff95ea0 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -40,8 +40,6 @@ #include -#include "videoconvert.h" - #include "gstvideoaggregator.h" #include "gstvideoaggregatorpad.h" @@ -66,7 +64,7 @@ enum struct _GstVideoAggregatorPadPrivate { /* Converter, if NULL no conversion is done */ - VideoConvert *convert; + GstVideoConverter *convert; }; G_DEFINE_TYPE (GstVideoAggregatorPad, gst_videoaggregator_pad, @@ -139,7 +137,7 @@ gst_videoaggregator_pad_finalize (GObject * o) GstVideoAggregatorPad *vaggpad = GST_VIDEO_AGGREGATOR_PAD (o); if (vaggpad->priv->convert) - badvideoconvert_convert_free (vaggpad->priv->convert); + gst_video_converter_free (vaggpad->priv->convert); vaggpad->priv->convert = NULL; G_OBJECT_CLASS (gst_videoaggregator_pad_parent_class)->finalize (o); @@ -434,7 +432,7 @@ gst_videoaggregator_update_converters (GstVideoAggregator * vagg) continue; if (pad->priv->convert) - badvideoconvert_convert_free (pad->priv->convert); + gst_video_converter_free (pad->priv->convert); pad->priv->convert = NULL; @@ -447,7 +445,8 @@ gst_videoaggregator_update_converters (GstVideoAggregator * vagg) GST_DEBUG_OBJECT (pad, "This pad will be converted from %d to %d", GST_VIDEO_INFO_FORMAT (&pad->info), GST_VIDEO_INFO_FORMAT (&best_info)); - pad->priv->convert = badvideoconvert_convert_new (&pad->info, &best_info); + pad->priv->convert = + gst_video_converter_new (&pad->info, &best_info, NULL); pad->need_conversion_update = TRUE; if (!pad->priv->convert) { g_free (colorimetry); @@ -1058,8 +1057,7 @@ prepare_frames (GstVideoAggregator * vagg, GstVideoAggregatorPad * pad) return FALSE; } - badvideoconvert_convert_convert (pad->priv->convert, converted_frame, - frame); + gst_video_converter_frame (pad->priv->convert, converted_frame, frame); pad->converted_buffer = converted_buf; gst_video_frame_unmap (frame); } else { From bc399a04039cac26d41225426cbec10052bb908a Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Fri, 26 Sep 2014 01:34:54 +1000 Subject: [PATCH 044/381] glvideomixer: fix blending with low-alpha sources We also need to apply the blend paramaters to the alpha channel otherwise the output of the blend will appear black at low alpha values (< 0.2). --- ext/gl/gstglvideomixer.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/ext/gl/gstglvideomixer.c b/ext/gl/gstglvideomixer.c index 9ffdcb6ad4..e04dec3a5d 100644 --- a/ext/gl/gstglvideomixer.c +++ b/ext/gl/gstglvideomixer.c @@ -649,9 +649,8 @@ gst_gl_video_mixer_callback (gpointer stuff) gl->EnableVertexAttribArray (attr_position_loc); gl->EnableVertexAttribArray (attr_texture_loc); - gl->BlendFuncSeparate (GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA, GL_ONE, - GL_ZERO); - gl->BlendEquationSeparate (GL_FUNC_ADD, GL_FUNC_ADD); + gl->BlendFunc (GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA); + gl->BlendEquation (GL_FUNC_ADD); gl->ActiveTexture (GL_TEXTURE0); gl->BindTexture (GL_TEXTURE_2D, in_tex); From 62a430ad21026f9b7d2e4b43a12bd971b5e65534 Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Sun, 28 Sep 2014 16:58:42 +1000 Subject: [PATCH 045/381] videoaggregator: don't create converters when the subclass doesn't want them fixes a critical with glvideomixer after 35bd1969f912cecda0fb5df9595a005e6ddf4e7a CRITICAL **: gst_video_converter_new: assertion 'in_info->width == out_info->width' failed --- gst-libs/gst/video/gstvideoaggregator.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index a09ff95ea0..069cf78f40 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -409,9 +409,14 @@ gst_videoaggregator_update_converters (GstVideoAggregator * vagg) return FALSE; } + vagg->info = best_info; + + /* short circuit */ + if (vagg_klass->disable_frame_conversion) + return TRUE; + best_colorimetry = gst_video_colorimetry_to_string (&(best_info.colorimetry)); best_chroma = gst_video_chroma_to_string (best_info.chroma_site); - vagg->info = best_info; GST_DEBUG_OBJECT (vagg, "The output format will now be : %d with colorimetry : %s and chroma : %s", From 65ed8ecf07134490fc10074cc9620599b0a62a1f Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Tue, 30 Sep 2014 20:39:36 +1000 Subject: [PATCH 046/381] glvideomixer: update element documentation --- ext/gl/gstglvideomixer.c | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/ext/gl/gstglvideomixer.c b/ext/gl/gstglvideomixer.c index e04dec3a5d..16843be090 100644 --- a/ext/gl/gstglvideomixer.c +++ b/ext/gl/gstglvideomixer.c @@ -21,10 +21,8 @@ /** * SECTION:element-glvideomixer * - * glmixer sub element. N gl sink pads to 1 source pad. - * N + 1 OpenGL contexts shared together. - * N <= 6 because the rendering is more a like a cube than a video_mixer - * Each opengl input stream is rendered on a cube face + * Composites a number of streams into a single output scene using OpenGL in + * a similar fashion to compositor and videomixer. * * * Examples @@ -37,7 +35,6 @@ * videotestsrc ! glfiltercube ! queue ! m. \ * videotestsrc ! gleffects effect=6 ! queue ! m. * ]| - * FBO (Frame Buffer Object) is required. * */ From 30f6b63c3d47f78d5975aa6f0338e4e4993908e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Wed, 1 Oct 2014 17:11:16 +0300 Subject: [PATCH 047/381] videoaggregator: GstVideoConverter currently can't rescale and will assert Leads to ugly assertions instead of properly erroring out: CRITICAL **: gst_video_converter_new: assertion 'in_info->width == out_info->width' failed --- gst-libs/gst/video/gstvideoaggregator.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index 069cf78f40..bd9095f3eb 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -450,8 +450,13 @@ gst_videoaggregator_update_converters (GstVideoAggregator * vagg) GST_DEBUG_OBJECT (pad, "This pad will be converted from %d to %d", GST_VIDEO_INFO_FORMAT (&pad->info), GST_VIDEO_INFO_FORMAT (&best_info)); - pad->priv->convert = - gst_video_converter_new (&pad->info, &best_info, NULL); + /* TODO: GstVideoConverter currently can't rescale! */ + if (pad->info.width == best_info.width && + pad->info.height == best_info.height && + pad->info.par_n == best_info.par_n && + pad->info.par_d == best_info.par_d) + pad->priv->convert = + gst_video_converter_new (&pad->info, &best_info, NULL); pad->need_conversion_update = TRUE; if (!pad->priv->convert) { g_free (colorimetry); From f4451b764f06f5db543dbb994d97a93f569799a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Wed, 1 Oct 2014 17:18:05 +0300 Subject: [PATCH 048/381] videoaggregator: Revert the last commit and handle resolutions differences properly This is about converting the format, not about converting any widths and heights. Subclasses are expected to handler different resolutions themselves, like the videomixers already do properly. --- gst-libs/gst/video/gstvideoaggregator.c | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index bd9095f3eb..00dd666c7d 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -447,16 +447,14 @@ gst_videoaggregator_update_converters (GstVideoAggregator * vagg) if (best_format != GST_VIDEO_INFO_FORMAT (&pad->info) || g_strcmp0 (colorimetry, best_colorimetry) || g_strcmp0 (chroma, best_chroma)) { + GstVideoInfo tmp_info = pad->info; + tmp_info.finfo = best_info.finfo; + GST_DEBUG_OBJECT (pad, "This pad will be converted from %d to %d", GST_VIDEO_INFO_FORMAT (&pad->info), GST_VIDEO_INFO_FORMAT (&best_info)); - /* TODO: GstVideoConverter currently can't rescale! */ - if (pad->info.width == best_info.width && - pad->info.height == best_info.height && - pad->info.par_n == best_info.par_n && - pad->info.par_d == best_info.par_d) - pad->priv->convert = - gst_video_converter_new (&pad->info, &best_info, NULL); + pad->priv->convert = + gst_video_converter_new (&pad->info, &tmp_info, NULL); pad->need_conversion_update = TRUE; if (!pad->priv->convert) { g_free (colorimetry); From 6daef6a2f892e8a13aed6080a6c784406f982375 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Wed, 1 Oct 2014 17:22:59 +0300 Subject: [PATCH 049/381] videoaggregator: Also copy over the chroma siting and colorimetry when deciding on a conversion --- gst-libs/gst/video/gstvideoaggregator.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index 00dd666c7d..39e95edd80 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -449,6 +449,8 @@ gst_videoaggregator_update_converters (GstVideoAggregator * vagg) g_strcmp0 (chroma, best_chroma)) { GstVideoInfo tmp_info = pad->info; tmp_info.finfo = best_info.finfo; + tmp_info.chroma_site = best_info.chroma_site; + tmp_info.colorimetry = best_info.colorimetry; GST_DEBUG_OBJECT (pad, "This pad will be converted from %d to %d", GST_VIDEO_INFO_FORMAT (&pad->info), From d2f4d20d85ef87f9319a0d328c63e8ab9cf772ee Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Thu, 18 Sep 2014 17:14:22 +0200 Subject: [PATCH 050/381] videoaggregator: Do not to release VIDEO_AGGREGATOR_LOCK while setting format info We should be able to always keep the VIDEO_AGGREGATOR_LOCK while negotiating caps, this patch introduce that change. That also implies that we do not need the SETCAPS_LOCK anymore because now VIDEO_AGGREGATOR_LOCK guarantees that setcaps is not called from several threads and the gst_aggregator_set_caps method is now protected. https://bugzilla.gnome.org/show_bug.cgi?id=735042 --- gst-libs/gst/video/gstvideoaggregator.c | 56 +++++++------------------ 1 file changed, 16 insertions(+), 40 deletions(-) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index 39e95edd80..de2d1af1f2 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -245,31 +245,11 @@ gst_videoaggregator_child_proxy_init (gpointer g_iface, gpointer iface_data) } G_STMT_END -#define GST_VIDEO_AGGREGATOR_GET_SETCAPS_LOCK(vagg) (&GST_VIDEO_AGGREGATOR(vagg)->priv->setcaps_lock) -#define GST_VIDEO_AGGREGATOR_SETCAPS_LOCK(vagg) G_STMT_START { \ - GST_LOG_OBJECT (vagg, "Taking SETCAPS lock from thread %p", \ - g_thread_self()); \ - g_mutex_lock(GST_VIDEO_AGGREGATOR_GET_SETCAPS_LOCK(vagg)); \ - GST_LOG_OBJECT (vagg, "Took SETCAPS lock from thread %p", \ - g_thread_self()); \ - } G_STMT_END - -#define GST_VIDEO_AGGREGATOR_SETCAPS_UNLOCK(vagg) G_STMT_START { \ - GST_LOG_OBJECT (vagg, "Releasing SETCAPS lock from thread %p", \ - g_thread_self()); \ - g_mutex_unlock(GST_VIDEO_AGGREGATOR_GET_SETCAPS_LOCK(vagg)); \ - GST_LOG_OBJECT (vagg, "Took SETCAPS lock from thread %p", \ - g_thread_self()); \ - } G_STMT_END - struct _GstVideoAggregatorPrivate { /* Lock to prevent the state to change while aggregating */ GMutex lock; - /* Lock to prevent two src setcaps from happening at the same time */ - GMutex setcaps_lock; - /* Current downstream segment */ GstClockTime ts_offset; guint64 nframes; @@ -360,6 +340,11 @@ _find_best_video_format (GstVideoAggregator * vagg, GstCaps * downstream_caps, g_hash_table_unref (formats_table); } +/* WITH GST_VIDEO_AGGREGATOR_LOCK TAKEN + * NOTE: After calling that method you **have to** call + * gst_videoaggregator_update_src_caps (without releasing + * the GST_VIDEO_AGGREGATOR_LOCK in between) + */ static gboolean gst_videoaggregator_update_converters (GstVideoAggregator * vagg) { @@ -476,6 +461,7 @@ gst_videoaggregator_update_converters (GstVideoAggregator * vagg) return TRUE; } +/* WITH GST_VIDEO_AGGREGATOR_LOCK TAKEN */ static gboolean gst_videoaggregator_src_setcaps (GstVideoAggregator * vagg, GstCaps * caps) { @@ -492,8 +478,6 @@ gst_videoaggregator_src_setcaps (GstVideoAggregator * vagg, GstCaps * caps) ret = TRUE; - GST_VIDEO_AGGREGATOR_LOCK (vagg); - if (GST_VIDEO_INFO_FPS_N (&vagg->info) != GST_VIDEO_INFO_FPS_N (&info) || GST_VIDEO_INFO_FPS_D (&vagg->info) != GST_VIDEO_INFO_FPS_D (&info)) { if (agg->segment.position != -1) { @@ -505,18 +489,21 @@ gst_videoaggregator_src_setcaps (GstVideoAggregator * vagg, GstCaps * caps) vagg->info = info; - GST_VIDEO_AGGREGATOR_UNLOCK (vagg); - if (vagg->priv->current_caps == NULL || gst_caps_is_equal (caps, vagg->priv->current_caps) == FALSE) { gst_caps_replace (&vagg->priv->current_caps, caps); + GST_VIDEO_AGGREGATOR_UNLOCK (vagg); + gst_aggregator_set_src_caps (agg, caps); + + GST_VIDEO_AGGREGATOR_LOCK (vagg); } done: return ret; } +/* WITH GST_VIDEO_AGGREGATOR_LOCK TAKEN */ static gboolean gst_videoaggregator_update_src_caps (GstVideoAggregator * vagg) { @@ -529,8 +516,6 @@ gst_videoaggregator_update_src_caps (GstVideoAggregator * vagg) GstVideoAggregatorClass *vagg_klass = (GstVideoAggregatorClass *) klass; GstAggregator *agg = GST_AGGREGATOR (vagg); - GST_VIDEO_AGGREGATOR_SETCAPS_LOCK (vagg); - GST_VIDEO_AGGREGATOR_LOCK (vagg); GST_OBJECT_LOCK (vagg); for (l = GST_ELEMENT (vagg)->sinkpads; l; l = l->next) { GstVideoAggregatorPad *mpad = l->data; @@ -596,7 +581,6 @@ gst_videoaggregator_update_src_caps (GstVideoAggregator * vagg) if (vagg_klass->update_info) { if (!vagg_klass->update_info (vagg, &info)) { ret = FALSE; - GST_VIDEO_AGGREGATOR_UNLOCK (vagg); goto done; } } @@ -619,7 +603,6 @@ gst_videoaggregator_update_src_caps (GstVideoAggregator * vagg) if (gst_caps_is_empty (caps)) { GST_DEBUG_OBJECT (vagg, "empty caps"); ret = FALSE; - GST_VIDEO_AGGREGATOR_UNLOCK (vagg); goto done; } @@ -638,21 +621,15 @@ gst_videoaggregator_update_src_caps (GstVideoAggregator * vagg) gst_caps_unref (caps); caps = gst_video_info_to_caps (&info); - GST_VIDEO_AGGREGATOR_UNLOCK (vagg); - if (gst_videoaggregator_src_setcaps (vagg, caps)) { if (vagg_klass->negotiated_caps) ret = GST_VIDEO_AGGREGATOR_GET_CLASS (vagg)->negotiated_caps (vagg, caps); } gst_caps_unref (caps); - } else { - GST_VIDEO_AGGREGATOR_UNLOCK (vagg); } done: - GST_VIDEO_AGGREGATOR_SETCAPS_UNLOCK (vagg); - return ret; } @@ -692,11 +669,12 @@ gst_videoaggregator_pad_sink_setcaps (GstPad * pad, GstObject * parent, vaggpad->info = info; ret = gst_videoaggregator_update_converters (vagg); - GST_VIDEO_AGGREGATOR_UNLOCK (vagg); if (ret) ret = gst_videoaggregator_update_src_caps (vagg); + GST_VIDEO_AGGREGATOR_UNLOCK (vagg); + beach: return ret; } @@ -1192,11 +1170,11 @@ gst_videoaggregator_aggregate (GstAggregator * agg) return GST_FLOW_NOT_NEGOTIATED; } + GST_VIDEO_AGGREGATOR_LOCK (vagg); + if (gst_pad_check_reconfigure (agg->srcpad)) gst_videoaggregator_update_src_caps (vagg); - GST_VIDEO_AGGREGATOR_LOCK (vagg); - if (agg->segment.position == -1) output_start_time = agg->segment.start; else @@ -1718,7 +1696,6 @@ gst_videoaggregator_release_pad (GstElement * element, GstPad * pad) gst_videoaggregator_update_converters (vagg); gst_buffer_replace (&vaggpad->buffer, NULL); update_caps = GST_VIDEO_INFO_FORMAT (&vagg->info) != GST_VIDEO_FORMAT_UNKNOWN; - GST_VIDEO_AGGREGATOR_UNLOCK (vagg); gst_child_proxy_child_removed (GST_CHILD_PROXY (vagg), G_OBJECT (vaggpad), GST_OBJECT_NAME (vaggpad)); @@ -1729,6 +1706,7 @@ gst_videoaggregator_release_pad (GstElement * element, GstPad * pad) if (update_caps) gst_videoaggregator_update_src_caps (vagg); + GST_VIDEO_AGGREGATOR_UNLOCK (vagg); return; } @@ -1850,7 +1828,6 @@ gst_videoaggregator_finalize (GObject * o) GstVideoAggregator *vagg = GST_VIDEO_AGGREGATOR (o); g_mutex_clear (&vagg->priv->lock); - g_mutex_clear (&vagg->priv->setcaps_lock); G_OBJECT_CLASS (gst_videoaggregator_parent_class)->finalize (o); } @@ -1946,7 +1923,6 @@ gst_videoaggregator_init (GstVideoAggregator * vagg) vagg->priv->current_caps = NULL; g_mutex_init (&vagg->priv->lock); - g_mutex_init (&vagg->priv->setcaps_lock); /* initialize variables */ gst_videoaggregator_reset (vagg); } From 9e122f644288ab048d5982c006a80f8130d218d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Mon, 6 Oct 2014 10:11:47 +0300 Subject: [PATCH 051/381] videoaggregator: Unmap and free GstVideoFrames as needed after conversion and errors --- gst-libs/gst/video/gstvideoaggregator.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index de2d1af1f2..ae4b9ea4be 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -1042,12 +1042,16 @@ prepare_frames (GstVideoAggregator * vagg, GstVideoAggregatorPad * pad) converted_buf, GST_MAP_READWRITE)) { GST_WARNING_OBJECT (vagg, "Could not map converted frame"); + g_slice_free (GstVideoFrame, converted_frame); + gst_video_frame_unmap (frame); + g_slice_free (GstVideoFrame, frame); return FALSE; } gst_video_converter_frame (pad->priv->convert, converted_frame, frame); pad->converted_buffer = converted_buf; gst_video_frame_unmap (frame); + g_slice_free (GstVideoFrame, frame); } else { converted_frame = frame; converted_buf = pad->buffer; From 47aea188261404cbae99e8aeeee5d6467bb422c3 Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Wed, 30 Jul 2014 11:57:46 +0200 Subject: [PATCH 052/381] videoaggregator: Make sure not to unref a NULL pointer --- gst-libs/gst/video/gstvideoaggregator.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index ae4b9ea4be..ff7ea3be3f 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -940,7 +940,8 @@ gst_videoaggregator_fill_queues (GstVideoAggregator * vagg, } else { gst_buffer_unref (buf); buf = gst_aggregator_pad_steal_buffer (bpad); - gst_buffer_unref (buf); + if (buf) + gst_buffer_unref (buf); } eos = FALSE; } else if (start_time >= output_end_time) { @@ -956,7 +957,8 @@ gst_videoaggregator_fill_queues (GstVideoAggregator * vagg, } else { gst_buffer_unref (buf); buf = gst_aggregator_pad_steal_buffer (bpad); - gst_buffer_unref (buf); + if (buf) + gst_buffer_unref (buf); } need_more_data = TRUE; From 138e63fd52d6461825b0ad5f095174661952bdb4 Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Tue, 23 Sep 2014 15:59:10 +0200 Subject: [PATCH 053/381] videoaggregator: Do not wrongly set the aggregator.segment The aggregator.segment is not to be initialized by the subclasses but by the aggregator itself. Moreover, initializing it on start would make us loose the information coming from the initial seek. --- gst-libs/gst/video/gstvideoaggregator.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index ff7ea3be3f..4464b40145 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -782,7 +782,6 @@ gst_videoaggregator_reset (GstVideoAggregator * vagg) vagg->priv->ts_offset = 0; vagg->priv->nframes = 0; - gst_segment_init (&agg->segment, GST_FORMAT_TIME); agg->segment.position = -1; gst_videoaggregator_reset_qos (vagg); @@ -1638,7 +1637,6 @@ gst_videoaggregator_start (GstAggregator * agg) if (!GST_AGGREGATOR_CLASS (gst_videoaggregator_parent_class)->start (agg)) return FALSE; - gst_segment_init (&agg->segment, GST_FORMAT_TIME); gst_caps_replace (&vagg->priv->current_caps, NULL); return TRUE; From 1e84482f360b5f354ee5c7310a5bc0c21d0d65fb Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Mon, 6 Oct 2014 18:10:38 +1100 Subject: [PATCH 054/381] videoaggregator: remove the use of the queued buffer on sink pads That data is now held by the aggregator class --- gst-libs/gst/video/gstvideoaggregator.c | 83 +++++++++---------------- 1 file changed, 29 insertions(+), 54 deletions(-) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index 4464b40145..865459d45d 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -791,7 +791,6 @@ gst_videoaggregator_reset (GstVideoAggregator * vagg) GstVideoAggregatorPad *p = l->data; gst_buffer_replace (&p->buffer, NULL); - gst_buffer_replace (&p->queued, NULL); p->start_time = -1; p->end_time = -1; @@ -810,6 +809,9 @@ gst_videoaggregator_fill_queues (GstVideoAggregator * vagg, gboolean eos = TRUE; gboolean need_more_data = FALSE; + /* get a set of buffers into pad->buffer that are within output_start_time + * and output_end_time taking into account finished and unresponsive pads */ + GST_OBJECT_LOCK (vagg); for (l = GST_ELEMENT (vagg)->sinkpads; l; l = l->next) { GstVideoAggregatorPad *pad = l->data; @@ -838,8 +840,8 @@ gst_videoaggregator_fill_queues (GstVideoAggregator * vagg, /* FIXME: Make all this work with negative rates */ - if ((pad->buffer && start_time < GST_BUFFER_TIMESTAMP (pad->buffer)) - || (pad->queued && start_time < GST_BUFFER_TIMESTAMP (pad->queued))) { + if ((start_time < GST_BUFFER_TIMESTAMP (buf)) + || (pad->buffer && start_time < GST_BUFFER_TIMESTAMP (pad->buffer))) { GST_DEBUG_OBJECT (pad, "Buffer from the past, dropping"); gst_buffer_unref (buf); buf = gst_aggregator_pad_steal_buffer (bpad); @@ -848,24 +850,15 @@ gst_videoaggregator_fill_queues (GstVideoAggregator * vagg, continue; } - if (pad->queued) { - end_time = start_time - GST_BUFFER_TIMESTAMP (pad->queued); - start_time = GST_BUFFER_TIMESTAMP (pad->queued); - gst_buffer_unref (buf); - buf = gst_buffer_ref (pad->queued); - vinfo = &pad->queued_vinfo; - } else { - end_time = GST_BUFFER_DURATION (buf); + end_time = GST_BUFFER_DURATION (buf); - if (end_time == -1) { - pad->queued = buf; - buf = gst_aggregator_pad_steal_buffer (bpad); - gst_buffer_unref (buf); - pad->queued_vinfo = pad->info; - GST_DEBUG ("end time is -1 and nothing queued"); - need_more_data = TRUE; - continue; - } + if (end_time == -1) { + gst_buffer_unref (buf); + buf = gst_aggregator_pad_steal_buffer (bpad); + gst_buffer_replace (&pad->buffer, buf); + gst_buffer_unref (buf); + GST_DEBUG_OBJECT (pad, "buffer duration is -1"); + continue; } g_assert (start_time != -1 && end_time != -1); @@ -880,14 +873,9 @@ gst_videoaggregator_fill_queues (GstVideoAggregator * vagg, GST_TIME_ARGS (segment->start), GST_TIME_ARGS (start_time), GST_TIME_ARGS (end_time)); - if (buf == pad->queued) { - gst_buffer_unref (buf); - gst_buffer_replace (&pad->queued, NULL); - } else { - gst_buffer_unref (buf); - buf = gst_aggregator_pad_steal_buffer (bpad); - gst_buffer_unref (buf); - } + gst_buffer_unref (buf); + buf = gst_aggregator_pad_steal_buffer (bpad); + gst_buffer_unref (buf); need_more_data = TRUE; continue; @@ -911,15 +899,9 @@ gst_videoaggregator_fill_queues (GstVideoAggregator * vagg, if (pad->end_time != -1 && pad->end_time > end_time) { GST_DEBUG_OBJECT (pad, "Buffer from the past, dropping"); - if (buf == pad->queued) { - gst_buffer_unref (buf); - gst_buffer_replace (&pad->queued, NULL); - } else { - gst_buffer_unref (buf); - buf = gst_aggregator_pad_steal_buffer (bpad); - gst_buffer_unref (buf); - } - + gst_buffer_unref (buf); + buf = gst_aggregator_pad_steal_buffer (bpad); + gst_buffer_unref (buf); need_more_data = TRUE; continue; } @@ -933,15 +915,10 @@ gst_videoaggregator_fill_queues (GstVideoAggregator * vagg, pad->start_time = start_time; pad->end_time = end_time; - if (buf == pad->queued) { + gst_buffer_unref (buf); + buf = gst_aggregator_pad_steal_buffer (bpad); + if (buf) gst_buffer_unref (buf); - gst_buffer_replace (&pad->queued, NULL); - } else { - gst_buffer_unref (buf); - buf = gst_aggregator_pad_steal_buffer (bpad); - if (buf) - gst_buffer_unref (buf); - } eos = FALSE; } else if (start_time >= output_end_time) { GST_DEBUG_OBJECT (pad, "Keeping buffer until %" GST_TIME_FORMAT, @@ -949,16 +926,14 @@ gst_videoaggregator_fill_queues (GstVideoAggregator * vagg, gst_buffer_unref (buf); eos = FALSE; } else { - GST_DEBUG_OBJECT (pad, "Too old buffer -- dropping"); - if (buf == pad->queued) { + GST_DEBUG_OBJECT (pad, + "Too old buffer -- dropping start %" GST_TIME_FORMAT " out end %" + GST_TIME_FORMAT, GST_TIME_ARGS (start_time), + GST_TIME_ARGS (output_end_time)); + gst_buffer_unref (buf); + buf = gst_aggregator_pad_steal_buffer (bpad); + if (buf) gst_buffer_unref (buf); - gst_buffer_replace (&pad->queued, NULL); - } else { - gst_buffer_unref (buf); - buf = gst_aggregator_pad_steal_buffer (bpad); - if (buf) - gst_buffer_unref (buf); - } need_more_data = TRUE; continue; From 9aa89185eda6e445cb1468e5614fdd8a02e59ad9 Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Mon, 6 Oct 2014 18:33:52 +1100 Subject: [PATCH 055/381] videoaggregator: support unresponsive pads Render unresponsive pads with the last video frame received. --- gst-libs/gst/video/gstvideoaggregator.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index 865459d45d..1edb4b958b 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -824,6 +824,8 @@ gst_videoaggregator_fill_queues (GstVideoAggregator * vagg, bpad = GST_AGGREGATOR_PAD (pad); segment = &bpad->segment; is_eos = bpad->eos; + if (!is_eos) + eos = FALSE; buf = gst_aggregator_pad_get_buffer (bpad); if (buf) { GstClockTime start_time, end_time; @@ -897,7 +899,8 @@ gst_videoaggregator_fill_queues (GstVideoAggregator * vagg, end_time *= ABS (agg->segment.rate); } - if (pad->end_time != -1 && pad->end_time > end_time) { + if (pad->end_time != -1 && pad->end_time > end_time + && !bpad->unresponsive) { GST_DEBUG_OBJECT (pad, "Buffer from the past, dropping"); gst_buffer_unref (buf); buf = gst_aggregator_pad_steal_buffer (bpad); @@ -939,9 +942,8 @@ gst_videoaggregator_fill_queues (GstVideoAggregator * vagg, continue; } } else { - if (pad->end_time != -1) { + if (!bpad->unresponsive && pad->end_time != -1) { if (pad->end_time <= output_start_time) { - gst_buffer_replace (&pad->buffer, NULL); pad->start_time = pad->end_time = -1; if (is_eos) { GST_DEBUG ("I just need more data"); @@ -950,6 +952,8 @@ gst_videoaggregator_fill_queues (GstVideoAggregator * vagg, } else if (is_eos) { eos = FALSE; } + } else if (is_eos) { + gst_buffer_replace (&pad->buffer, NULL); } } } From b30f8985d5eee335c787646309b576751e1ab1b3 Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Mon, 6 Oct 2014 21:46:24 +1100 Subject: [PATCH 056/381] aggregator: add latency query handling --- gst-libs/gst/video/gstvideoaggregator.c | 83 ------------------------- 1 file changed, 83 deletions(-) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index 1edb4b958b..8f46b7b790 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -1324,87 +1324,6 @@ gst_videoaggregator_query_duration (GstVideoAggregator * vagg, GstQuery * query) return res; } -static gboolean -gst_videoaggregator_query_latency (GstVideoAggregator * vagg, GstQuery * query) -{ - GstClockTime min, max; - gboolean live; - gboolean res; - GstIterator *it; - gboolean done; - GValue item = { 0 }; - - res = TRUE; - done = FALSE; - live = FALSE; - min = 0; - max = GST_CLOCK_TIME_NONE; - - /* Take maximum of all latency values */ - it = gst_element_iterate_sink_pads (GST_ELEMENT_CAST (vagg)); - while (!done) { - switch (gst_iterator_next (it, &item)) { - case GST_ITERATOR_DONE: - done = TRUE; - break; - case GST_ITERATOR_OK: - { - GstPad *pad = g_value_get_object (&item); - GstQuery *peerquery; - GstClockTime min_cur, max_cur; - gboolean live_cur; - - peerquery = gst_query_new_latency (); - - /* Ask peer for latency */ - res &= gst_pad_peer_query (pad, peerquery); - - /* take max from all valid return values */ - if (res) { - gst_query_parse_latency (peerquery, &live_cur, &min_cur, &max_cur); - - if (min_cur > min) - min = min_cur; - - if (max_cur != GST_CLOCK_TIME_NONE && - ((max != GST_CLOCK_TIME_NONE && max_cur > max) || - (max == GST_CLOCK_TIME_NONE))) - max = max_cur; - - live = live || live_cur; - } - - gst_query_unref (peerquery); - g_value_reset (&item); - break; - } - case GST_ITERATOR_RESYNC: - live = FALSE; - min = 0; - max = GST_CLOCK_TIME_NONE; - res = TRUE; - gst_iterator_resync (it); - break; - default: - res = FALSE; - done = TRUE; - break; - } - } - g_value_unset (&item); - gst_iterator_free (it); - - if (res) { - /* store the results */ - GST_DEBUG_OBJECT (vagg, "Calculated total latency: live %s, min %" - GST_TIME_FORMAT ", max %" GST_TIME_FORMAT, - (live ? "yes" : "no"), GST_TIME_ARGS (min), GST_TIME_ARGS (max)); - gst_query_set_latency (query, live, min, max); - } - - return res; -} - static gboolean gst_videoaggregator_src_query (GstAggregator * agg, GstQuery * query) { @@ -1434,8 +1353,6 @@ gst_videoaggregator_src_query (GstAggregator * agg, GstQuery * query) res = gst_videoaggregator_query_duration (vagg, query); break; case GST_QUERY_LATENCY: - res = gst_videoaggregator_query_latency (vagg, query); - break; case GST_QUERY_CAPS: res = GST_AGGREGATOR_CLASS (gst_videoaggregator_parent_class)->src_query From a92ffca74566077271dfc857b69ab313b2e1f77b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Mon, 13 Oct 2014 11:53:47 +0200 Subject: [PATCH 057/381] glmixer: Add other-context property --- ext/gl/gstglmixer.c | 41 +++++++++++++++++++++++++++++++++++++++-- ext/gl/gstglmixer.h | 2 +- 2 files changed, 40 insertions(+), 3 deletions(-) diff --git a/ext/gl/gstglmixer.c b/ext/gl/gstglmixer.c index 04c951fdb5..0095437dcf 100644 --- a/ext/gl/gstglmixer.c +++ b/ext/gl/gstglmixer.c @@ -182,7 +182,7 @@ gst_gl_mixer_propose_allocation (GstGLMixer * mix, if (!mix->context) { mix->context = gst_gl_context_new (mix->display); - if (!gst_gl_context_create (mix->context, NULL, &error)) + if (!gst_gl_context_create (mix->context, mix->other_context, &error)) goto context_error; } @@ -327,7 +327,8 @@ enum enum { - PROP_0 + PROP_0, + PROP_OTHER_CONTEXT }; static GstStaticPadTemplate src_factory = GST_STATIC_PAD_TEMPLATE ("src", @@ -403,6 +404,12 @@ gst_gl_mixer_class_init (GstGLMixerClass * klass) gobject_class->get_property = gst_gl_mixer_get_property; gobject_class->set_property = gst_gl_mixer_set_property; + g_object_class_install_property (gobject_class, PROP_OTHER_CONTEXT, + g_param_spec_object ("other-context", + "External OpenGL context", + "Give an external OpenGL context with which to share textures", + GST_GL_TYPE_CONTEXT, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + gst_element_class_add_pad_template (element_class, gst_static_pad_template_get (&src_factory)); gst_element_class_add_pad_template (element_class, @@ -642,6 +649,16 @@ gst_gl_mixer_decide_allocation (GstGLMixer * mix, GstQuery * query) } } + if (mix->other_context) { + if (!other_context) { + other_context = mix->other_context; + } else { + GST_ELEMENT_WARNING (mix, LIBRARY, SETTINGS, + ("%s", "Cannot share with more than one GL context"), + ("%s", "Cannot share with more than one GL context")); + } + } + if (!mix->context) { mix->context = gst_gl_context_new (mix->display); if (!gst_gl_context_create (mix->context, other_context, &error)) @@ -998,7 +1015,12 @@ static void gst_gl_mixer_get_property (GObject * object, guint prop_id, GValue * value, GParamSpec * pspec) { + GstGLMixer *mix = GST_GL_MIXER (object); + switch (prop_id) { + case PROP_OTHER_CONTEXT: + g_value_set_object (value, mix->other_context); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); break; @@ -1009,7 +1031,16 @@ static void gst_gl_mixer_set_property (GObject * object, guint prop_id, const GValue * value, GParamSpec * pspec) { + GstGLMixer *mix = GST_GL_MIXER (object); + switch (prop_id) { + case PROP_OTHER_CONTEXT: + { + if (mix->other_context) + gst_object_unref (mix->other_context); + mix->other_context = g_value_dup_object (value); + break; + } default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); break; @@ -1110,6 +1141,12 @@ gst_gl_mixer_stop (GstAggregator * agg) gst_object_unref (mix->context); mix->context = NULL; } + + if (mix->other_context) { + gst_object_unref (mix->other_context); + mix->other_context = NULL; + } + gst_gl_mixer_reset (mix); return TRUE; diff --git a/ext/gl/gstglmixer.h b/ext/gl/gstglmixer.h index d66ef30c18..5ea246ba5e 100644 --- a/ext/gl/gstglmixer.h +++ b/ext/gl/gstglmixer.h @@ -66,7 +66,7 @@ struct _GstGLMixer GstGLDownload *download; GstGLDisplay *display; - GstGLContext *context; + GstGLContext *context, *other_context; GLuint fbo; GLuint depthbuffer; From 21275caae58965ffcc5d14d85ea919fbfda78298 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Mon, 13 Oct 2014 13:27:11 +0200 Subject: [PATCH 058/381] glmixer: Only finalize the other context in finalize() Otherwise we change a value of a property when going to READY state, which is unexpected behaviour. --- ext/gl/gstglmixer.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/ext/gl/gstglmixer.c b/ext/gl/gstglmixer.c index 0095437dcf..5857410599 100644 --- a/ext/gl/gstglmixer.c +++ b/ext/gl/gstglmixer.c @@ -463,7 +463,13 @@ gst_gl_mixer_init (GstGLMixer * mix) static void gst_gl_mixer_finalize (GObject * object) { - GstGLMixerPrivate *priv = GST_GL_MIXER (object)->priv; + GstGLMixer *mix = GST_GL_MIXER (object); + GstGLMixerPrivate *priv = mix->priv; + + if (mix->other_context) { + gst_object_unref (mix->other_context); + mix->other_context = NULL; + } g_mutex_clear (&priv->gl_resource_lock); g_cond_clear (&priv->gl_resource_cond); @@ -1142,11 +1148,6 @@ gst_gl_mixer_stop (GstAggregator * agg) mix->context = NULL; } - if (mix->other_context) { - gst_object_unref (mix->other_context); - mix->other_context = NULL; - } - gst_gl_mixer_reset (mix); return TRUE; From 318b55e3ae69a36e1f2f0d1ac287e4922ffcda98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Mon, 13 Oct 2014 13:27:55 +0200 Subject: [PATCH 059/381] glmixer: Call the pad's parent class finalize method --- ext/gl/gstglmixer.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ext/gl/gstglmixer.c b/ext/gl/gstglmixer.c index 5857410599..0a299db88a 100644 --- a/ext/gl/gstglmixer.c +++ b/ext/gl/gstglmixer.c @@ -99,6 +99,8 @@ gst_gl_mixer_pad_finalize (GObject * object) gst_object_unref (pad->upload); pad->upload = NULL; } + + G_OBJECT_CLASS (gst_gl_mixer_pad_parent_class)->finalize (object); } static void From 8bf2acba1d2a81ded284a3f6092972a2c20b9cc5 Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Mon, 20 Oct 2014 10:34:27 +1100 Subject: [PATCH 060/381] videoaggregator: operate on caps rather than video info Otherwise the CapsFeatures will be lost along with the possibility of multiple output types and formats. https://bugzilla.gnome.org/show_bug.cgi?id=738129 --- ext/gl/gstglmixer.c | 96 +++++++++++++++++++++++++ ext/gl/gstglmixer.h | 1 + ext/gl/gstglvideomixer.c | 26 ++++--- gst-libs/gst/video/gstvideoaggregator.c | 38 ++++++---- gst-libs/gst/video/gstvideoaggregator.h | 10 +-- gst/compositor/compositor.c | 18 +++-- 6 files changed, 155 insertions(+), 34 deletions(-) diff --git a/ext/gl/gstglmixer.c b/ext/gl/gstglmixer.c index 0a299db88a..37aa8f0b43 100644 --- a/ext/gl/gstglmixer.c +++ b/ext/gl/gstglmixer.c @@ -36,6 +36,10 @@ #include "gstglmixer.h" +#if GST_GL_HAVE_PLATFORM_EGL +#include +#endif + #define gst_gl_mixer_parent_class parent_class G_DEFINE_ABSTRACT_TYPE (GstGLMixer, gst_gl_mixer, GST_TYPE_VIDEO_AGGREGATOR); static gboolean gst_gl_mixer_do_bufferpool (GstGLMixer * mix, @@ -559,6 +563,98 @@ gst_gl_mixer_query_caps (GstPad * pad, GstAggregator * agg, GstQuery * query) return TRUE; } +static GstCaps * +gst_gl_mixer_set_caps_features (const GstCaps * caps, + const gchar * feature_name) +{ + GstCaps *tmp = gst_caps_copy (caps); + guint n = gst_caps_get_size (tmp); + guint i = 0; + + for (i = 0; i < n; i++) { + GstCapsFeatures *features = gst_caps_get_features (tmp, i); + if (features) { + guint n_f = gst_caps_features_get_size (features); + guint j = 0; + for (j = 0; j < n_f; j++) { + gst_caps_features_remove_id (features, + gst_caps_features_get_nth_id (features, j)); + } + } + + gst_caps_features_add (features, feature_name); + gst_caps_set_simple (tmp, "format", G_TYPE_STRING, "RGBA", NULL); + } + + return tmp; +} + +/* copies the given caps */ +static GstCaps * +gst_gl_mixer_caps_remove_format_info (GstCaps * caps) +{ + GstStructure *st; + GstCapsFeatures *f; + gint i, n; + GstCaps *res; + + res = gst_caps_new_empty (); + + n = gst_caps_get_size (caps); + for (i = 0; i < n; i++) { + st = gst_caps_get_structure (caps, i); + f = gst_caps_get_features (caps, i); + + /* If this is already expressed by the existing caps + * skip this structure */ + if (i > 0 && gst_caps_is_subset_structure_full (res, st, f)) + continue; + + st = gst_structure_copy (st); + /* Only remove format info for the cases when we can actually convert */ + if (!gst_caps_features_is_any (f) + && gst_caps_features_is_equal (f, + GST_CAPS_FEATURES_MEMORY_SYSTEM_MEMORY)) + gst_structure_remove_fields (st, "format", "colorimetry", "chroma-site", + "width", "height", NULL); + + gst_caps_append_structure_full (res, st, gst_caps_features_copy (f)); + } + + return res; +} + +GstCaps * +gst_gl_mixer_update_caps (GstGLMixer * mix, GstCaps * caps) +{ + GstCaps *result = NULL; + GstCaps *glcaps = gst_gl_mixer_set_caps_features (caps, + GST_CAPS_FEATURE_MEMORY_GL_MEMORY); +#if GST_GL_HAVE_PLATFORM_EGL + GstCaps *eglcaps = gst_gl_mixer_set_caps_features (caps, + GST_CAPS_FEATURE_MEMORY_EGL_IMAGE); +#endif + GstCaps *uploadcaps = gst_gl_mixer_set_caps_features (caps, + GST_CAPS_FEATURE_META_GST_VIDEO_GL_TEXTURE_UPLOAD_META); + GstCaps *raw_caps = + gst_caps_from_string (GST_VIDEO_CAPS_MAKE (GST_GL_COLOR_CONVERT_FORMATS)); + + result = gst_caps_new_empty (); + + result = gst_caps_merge (result, glcaps); +#if GST_GL_HAVE_PLATFORM_EGL + result = gst_caps_merge (result, eglcaps); +#endif + result = gst_caps_merge (result, uploadcaps); + result = gst_caps_merge (result, raw_caps); + + result = gst_caps_merge (result, gst_gl_mixer_caps_remove_format_info (caps)); + + GST_DEBUG_OBJECT (mix, "returning %" GST_PTR_FORMAT, result); + + return result; +} + static gboolean gst_gl_mixer_src_query (GstAggregator * agg, GstQuery * query) { diff --git a/ext/gl/gstglmixer.h b/ext/gl/gstglmixer.h index 5ea246ba5e..f25c30a32f 100644 --- a/ext/gl/gstglmixer.h +++ b/ext/gl/gstglmixer.h @@ -92,6 +92,7 @@ struct _GstGLMixerFrameData GType gst_gl_mixer_get_type(void); gboolean gst_gl_mixer_process_textures (GstGLMixer * mix, GstBuffer * outbuf); +GstCaps * gst_gl_mixer_update_caps (GstGLMixer * mix, GstCaps * caps); G_END_DECLS #endif /* __GST_GL_MIXER_H__ */ diff --git a/ext/gl/gstglvideomixer.c b/ext/gl/gstglvideomixer.c index 16843be090..39e17ae21d 100644 --- a/ext/gl/gstglvideomixer.c +++ b/ext/gl/gstglvideomixer.c @@ -65,7 +65,7 @@ static void gst_gl_video_mixer_set_property (GObject * object, guint prop_id, static void gst_gl_video_mixer_get_property (GObject * object, guint prop_id, GValue * value, GParamSpec * pspec); -static gboolean _update_info (GstVideoAggregator * vagg, GstVideoInfo * info); +static GstCaps *_update_caps (GstVideoAggregator * vagg, GstCaps * caps); static void gst_gl_video_mixer_reset (GstGLMixer * mixer); static gboolean gst_gl_video_mixer_init_shader (GstGLMixer * mixer, GstCaps * outcaps); @@ -335,7 +335,7 @@ gst_gl_video_mixer_class_init (GstGLVideoMixerClass * klass) GST_GL_MIXER_CLASS (klass)->process_textures = gst_gl_video_mixer_process_textures; - vagg_class->update_info = _update_info; + vagg_class->update_caps = _update_caps; agg_class->sinkpads_type = GST_TYPE_GL_VIDEO_MIXER_PAD; } @@ -380,12 +380,17 @@ gst_gl_video_mixer_get_property (GObject * object, guint prop_id, } } -static gboolean -_update_info (GstVideoAggregator * vagg, GstVideoInfo * info) +static GstCaps * +_update_caps (GstVideoAggregator * vagg, GstCaps * caps) { GList *l; gint best_width = -1, best_height = -1; - gboolean ret = FALSE; + GstVideoInfo info; + GstCaps *ret = NULL; + int i; + + caps = gst_caps_make_writable (caps); + gst_video_info_from_caps (&info, caps); GST_OBJECT_LOCK (vagg); for (l = GST_ELEMENT (vagg)->sinkpads; l; l = l->next) { @@ -417,10 +422,13 @@ _update_info (GstVideoAggregator * vagg, GstVideoInfo * info) } GST_OBJECT_UNLOCK (vagg); - if (best_width > 0 && best_height > 0) { - info->width = best_width; - info->height = best_height; - ret = TRUE; + ret = gst_gl_mixer_update_caps (GST_GL_MIXER (vagg), caps); + + for (i = 0; i < gst_caps_get_size (ret); i++) { + GstStructure *s = gst_caps_get_structure (ret, i); + + gst_structure_set (s, "width", G_TYPE_INT, best_width, "height", G_TYPE_INT, + best_height, NULL); } return ret; diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index 8f46b7b790..83bf08be69 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -559,7 +559,7 @@ gst_videoaggregator_update_src_caps (GstVideoAggregator * vagg) } if (best_width > 0 && best_height > 0 && best_fps > 0) { - GstCaps *caps, *peercaps; + GstCaps *caps, *peercaps, *info_caps; GstStructure *s; GstVideoInfo info; @@ -578,25 +578,40 @@ gst_videoaggregator_update_src_caps (GstVideoAggregator * vagg) info.par_n = GST_VIDEO_INFO_PAR_N (&vagg->info); info.par_d = GST_VIDEO_INFO_PAR_D (&vagg->info); - if (vagg_klass->update_info) { - if (!vagg_klass->update_info (vagg, &info)) { + info_caps = gst_video_info_to_caps (&info); + + if (vagg_klass->update_caps) { + if (!(caps = vagg_klass->update_caps (vagg, info_caps))) { + gst_caps_unref (info_caps); ret = FALSE; goto done; } + gst_caps_unref (info_caps); + } else { + caps = info_caps; } - caps = gst_video_info_to_caps (&info); - peercaps = gst_pad_peer_query_caps (agg->srcpad, NULL); if (peercaps) { GstCaps *tmp; + int i; s = gst_caps_get_structure (caps, 0); - gst_structure_set (s, "width", GST_TYPE_INT_RANGE, 1, G_MAXINT, "height", - GST_TYPE_INT_RANGE, 1, G_MAXINT, "framerate", GST_TYPE_FRACTION_RANGE, - 0, 1, G_MAXINT, 1, NULL); + gst_structure_get (s, "width", G_TYPE_INT, &best_width, "height", + G_TYPE_INT, &best_height, NULL); + + for (i = 0; i < gst_caps_get_size (caps); i++) { + s = gst_caps_get_structure (caps, i); + gst_structure_set (s, "width", GST_TYPE_INT_RANGE, 1, G_MAXINT, + "height", GST_TYPE_INT_RANGE, 1, G_MAXINT, "framerate", + GST_TYPE_FRACTION_RANGE, 0, 1, G_MAXINT, 1, NULL); + } tmp = gst_caps_intersect (caps, peercaps); + GST_DEBUG_OBJECT (vagg, "intersecting %" GST_PTR_FORMAT + " with peer caps %" GST_PTR_FORMAT " result %" GST_PTR_FORMAT, caps, + peercaps, tmp); + gst_caps_unref (caps); gst_caps_unref (peercaps); caps = tmp; @@ -608,8 +623,8 @@ gst_videoaggregator_update_src_caps (GstVideoAggregator * vagg) caps = gst_caps_truncate (caps); s = gst_caps_get_structure (caps, 0); - gst_structure_fixate_field_nearest_int (s, "width", info.width); - gst_structure_fixate_field_nearest_int (s, "height", info.height); + gst_structure_fixate_field_nearest_int (s, "width", best_width); + gst_structure_fixate_field_nearest_int (s, "height", best_height); gst_structure_fixate_field_nearest_fraction (s, "framerate", best_fps_n, best_fps_d); @@ -618,9 +633,6 @@ gst_videoaggregator_update_src_caps (GstVideoAggregator * vagg) gst_structure_get_fraction (s, "framerate", &info.fps_n, &info.fps_d); } - gst_caps_unref (caps); - caps = gst_video_info_to_caps (&info); - if (gst_videoaggregator_src_setcaps (vagg, caps)) { if (vagg_klass->negotiated_caps) ret = diff --git a/gst-libs/gst/video/gstvideoaggregator.h b/gst-libs/gst/video/gstvideoaggregator.h index b9fd53164c..2bdd6195f8 100644 --- a/gst-libs/gst/video/gstvideoaggregator.h +++ b/gst-libs/gst/video/gstvideoaggregator.h @@ -73,9 +73,9 @@ struct _GstVideoAggregator * @disable_frame_conversion: Optional. * Allows subclasses to disable the frame colorspace * conversion feature - * @update_info: Optional. - * Lets subclasses update the src #GstVideoInfo representing - * the src pad caps before usage. + * @update_caps: Optional. + * Lets subclasses update the #GstCaps representing + * the src pad caps before usage. Return %NULL to indicate failure. * @aggregate_frames: Lets subclasses aggregate frames that are ready. Subclasses * should iterate the GstElement.sinkpads and use the already * mapped #GstVideoFrame from GstVideoAggregatorPad.aggregated_frame @@ -96,8 +96,8 @@ struct _GstVideoAggregatorClass /*< public >*/ gboolean disable_frame_conversion; - gboolean (*update_info) (GstVideoAggregator * videoaggregator, - GstVideoInfo * info); + GstCaps * (*update_caps) (GstVideoAggregator * videoaggregator, + GstCaps * caps); GstFlowReturn (*aggregate_frames) (GstVideoAggregator * videoaggregator, GstBuffer * outbuffer); GstFlowReturn (*get_output_buffer) (GstVideoAggregator * videoaggregator, diff --git a/gst/compositor/compositor.c b/gst/compositor/compositor.c index 805b674543..8b68950c23 100644 --- a/gst/compositor/compositor.c +++ b/gst/compositor/compositor.c @@ -437,12 +437,15 @@ set_functions (GstCompositor * self, GstVideoInfo * info) return ret; } -static gboolean -_update_info (GstVideoAggregator * vagg, GstVideoInfo * info) +static GstCaps * +_update_caps (GstVideoAggregator * vagg, GstCaps * caps) { GList *l; gint best_width = -1, best_height = -1; - gboolean ret = FALSE; + GstVideoInfo info; + GstCaps *ret = NULL; + + gst_video_info_from_caps (&info, caps); GST_OBJECT_LOCK (vagg); for (l = GST_ELEMENT (vagg)->sinkpads; l; l = l->next) { @@ -468,9 +471,10 @@ _update_info (GstVideoAggregator * vagg, GstVideoInfo * info) GST_OBJECT_UNLOCK (vagg); if (best_width > 0 && best_height > 0) { - gst_video_info_set_format (info, GST_VIDEO_INFO_FORMAT (info), - best_width, best_height); - ret = set_functions (GST_COMPOSITOR (vagg), info); + info.width = best_width; + info.height = best_height; + if (set_functions (GST_COMPOSITOR (vagg), &info)) + ret = gst_video_info_to_caps (&info); } return ret; @@ -559,7 +563,7 @@ gst_compositor_class_init (GstCompositorClass * klass) gobject_class->set_property = gst_compositor_set_property; agg_class->sinkpads_type = GST_TYPE_COMPOSITOR_PAD; - videoaggregator_class->update_info = _update_info; + videoaggregator_class->update_caps = _update_caps; videoaggregator_class->aggregate_frames = gst_compositor_aggregate_frames; g_object_class_install_property (gobject_class, PROP_BACKGROUND, From 7b4db67f6cae11e0874edd5316fe2d2d039ed002 Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Sat, 18 Oct 2014 10:03:43 +0200 Subject: [PATCH 061/381] glbufferpool: add the GstVideoGLTextureUploadMeta buffer pool option --- ext/gl/gstglmixer.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ext/gl/gstglmixer.c b/ext/gl/gstglmixer.c index 37aa8f0b43..fba4407e17 100644 --- a/ext/gl/gstglmixer.c +++ b/ext/gl/gstglmixer.c @@ -822,6 +822,8 @@ gst_gl_mixer_decide_allocation (GstGLMixer * mix, GstQuery * query) gst_buffer_pool_config_set_params (config, caps, size, min, max); gst_buffer_pool_config_add_option (config, GST_BUFFER_POOL_OPTION_VIDEO_META); + gst_buffer_pool_config_add_option (config, + GST_BUFFER_POOL_OPTION_VIDEO_GL_TEXTURE_UPLOAD_META); gst_buffer_pool_set_config (pool, config); From 165cdd20a73c19254a4b7c51bc6547e0c5229ea9 Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Tue, 21 Oct 2014 02:52:05 +1100 Subject: [PATCH 062/381] videoaggregator: fixate the parts of the caps we don't know how to deal with fixes glvideomixer with video/x-raw,width=foo i.e. no format field. --- gst-libs/gst/video/gstvideoaggregator.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index 83bf08be69..e24bb212e4 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -628,6 +628,9 @@ gst_videoaggregator_update_src_caps (GstVideoAggregator * vagg) gst_structure_fixate_field_nearest_fraction (s, "framerate", best_fps_n, best_fps_d); + /* fixate the the rest of the fields */ + caps = gst_caps_fixate (caps); + gst_structure_get_int (s, "width", &info.width); gst_structure_get_int (s, "height", &info.height); gst_structure_get_fraction (s, "framerate", &info.fps_n, &info.fps_d); From be214eac0fbaf08ec86cc046e61e3463b198381f Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Tue, 21 Oct 2014 06:04:10 +1100 Subject: [PATCH 063/381] glmixer: fixup eglimage include path --- ext/gl/gstglmixer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/gl/gstglmixer.c b/ext/gl/gstglmixer.c index fba4407e17..9f1ff132bf 100644 --- a/ext/gl/gstglmixer.c +++ b/ext/gl/gstglmixer.c @@ -37,7 +37,7 @@ #include "gstglmixer.h" #if GST_GL_HAVE_PLATFORM_EGL -#include +#include #endif #define gst_gl_mixer_parent_class parent_class From 340542edc73d3a372efe3d895448cf0f7df87e3d Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Thu, 14 Aug 2014 23:29:00 +1000 Subject: [PATCH 064/381] gl: propogate other-context using GstContext --- ext/gl/gstglmixer.c | 35 +++++++---------------------------- 1 file changed, 7 insertions(+), 28 deletions(-) diff --git a/ext/gl/gstglmixer.c b/ext/gl/gstglmixer.c index 9f1ff132bf..743bacc90b 100644 --- a/ext/gl/gstglmixer.c +++ b/ext/gl/gstglmixer.c @@ -183,7 +183,7 @@ gst_gl_mixer_propose_allocation (GstGLMixer * mix, gst_structure_free (config); } - if (!gst_gl_ensure_display (mix, &mix->display)) + if (!gst_gl_ensure_element_data (mix, &mix->display, &mix->other_context)) return FALSE; if (!mix->context) { @@ -308,7 +308,7 @@ gst_gl_mixer_sink_query (GstAggregator * agg, GstAggregatorPad * bpad, case GST_QUERY_CONTEXT: { ret = gst_gl_handle_context_query ((GstElement *) mix, query, - &mix->display); + &mix->display, &mix->other_context); break; } default: @@ -333,8 +333,7 @@ enum enum { - PROP_0, - PROP_OTHER_CONTEXT + PROP_0 }; static GstStaticPadTemplate src_factory = GST_STATIC_PAD_TEMPLATE ("src", @@ -410,12 +409,6 @@ gst_gl_mixer_class_init (GstGLMixerClass * klass) gobject_class->get_property = gst_gl_mixer_get_property; gobject_class->set_property = gst_gl_mixer_set_property; - g_object_class_install_property (gobject_class, PROP_OTHER_CONTEXT, - g_param_spec_object ("other-context", - "External OpenGL context", - "Give an external OpenGL context with which to share textures", - GST_GL_TYPE_CONTEXT, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); - gst_element_class_add_pad_template (element_class, gst_static_pad_template_get (&src_factory)); gst_element_class_add_pad_template (element_class, @@ -487,7 +480,7 @@ gst_gl_mixer_set_context (GstElement * element, GstContext * context) { GstGLMixer *mix = GST_GL_MIXER (element); - gst_gl_handle_set_context (element, context, &mix->display); + gst_gl_handle_set_context (element, context, &mix->display, &mix->context); } static gboolean @@ -496,7 +489,7 @@ gst_gl_mixer_activate (GstGLMixer * mix, gboolean active) gboolean result = TRUE; if (active) { - if (!gst_gl_ensure_display (mix, &mix->display)) + if (!gst_gl_ensure_element_data (mix, &mix->display, &mix->other_context)) result = FALSE; } @@ -665,7 +658,7 @@ gst_gl_mixer_src_query (GstAggregator * agg, GstQuery * query) case GST_QUERY_CONTEXT: { res = gst_gl_handle_context_query ((GstElement *) mix, query, - &mix->display); + &mix->display, &mix->other_context); break; } case GST_QUERY_CAPS: @@ -712,7 +705,7 @@ gst_gl_mixer_decide_allocation (GstGLMixer * mix, GstQuery * query) GstGLContext *other_context = NULL; GstVideoAggregator *vagg = GST_VIDEO_AGGREGATOR (mix); - if (!gst_gl_ensure_display (mix, &mix->display)) + if (!gst_gl_ensure_element_data (mix, &mix->display, &mix->other_context)) return FALSE; if (gst_query_find_allocation_meta (query, @@ -1121,12 +1114,7 @@ static void gst_gl_mixer_get_property (GObject * object, guint prop_id, GValue * value, GParamSpec * pspec) { - GstGLMixer *mix = GST_GL_MIXER (object); - switch (prop_id) { - case PROP_OTHER_CONTEXT: - g_value_set_object (value, mix->other_context); - break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); break; @@ -1137,16 +1125,7 @@ static void gst_gl_mixer_set_property (GObject * object, guint prop_id, const GValue * value, GParamSpec * pspec) { - GstGLMixer *mix = GST_GL_MIXER (object); - switch (prop_id) { - case PROP_OTHER_CONTEXT: - { - if (mix->other_context) - gst_object_unref (mix->other_context); - mix->other_context = g_value_dup_object (value); - break; - } default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); break; From 6687d397343fc0632d4237a62f69c398333fb41c Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Thu, 23 Oct 2014 23:55:48 +1100 Subject: [PATCH 065/381] glmixer: override the accept caps query in order to 'convert' capsfeatures Otherwise, it is only possible for the sink pads and the src pads to have the exact same caps features. We can convert from any feature to another feature so support that. --- ext/gl/gstglmixer.c | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/ext/gl/gstglmixer.c b/ext/gl/gstglmixer.c index 743bacc90b..da6ba67ed0 100644 --- a/ext/gl/gstglmixer.c +++ b/ext/gl/gstglmixer.c @@ -267,6 +267,26 @@ context_error: } } +static gboolean +gst_gl_mixer_pad_sink_acceptcaps (GstPad * pad, GstGLMixer * mix, + GstCaps * caps) +{ + gboolean ret; + GstCaps *template_caps; + + GST_DEBUG_OBJECT (pad, "try accept caps of %" GST_PTR_FORMAT, caps); + + template_caps = gst_pad_get_pad_template_caps (pad); + template_caps = gst_caps_make_writable (template_caps); + + ret = gst_caps_can_intersect (caps, template_caps); + GST_DEBUG_OBJECT (pad, "%saccepted caps %" GST_PTR_FORMAT, + (ret ? "" : "not "), caps); + gst_caps_unref (template_caps); + + return ret; +} + static gboolean gst_gl_mixer_sink_query (GstAggregator * agg, GstAggregatorPad * bpad, GstQuery * query) @@ -277,6 +297,16 @@ gst_gl_mixer_sink_query (GstAggregator * agg, GstAggregatorPad * bpad, GST_TRACE ("QUERY %" GST_PTR_FORMAT, query); switch (GST_QUERY_TYPE (query)) { + case GST_QUERY_ACCEPT_CAPS: + { + GstCaps *caps; + + gst_query_parse_accept_caps (query, &caps); + ret = gst_gl_mixer_pad_sink_acceptcaps (GST_PAD (bpad), mix, caps); + gst_query_set_accept_caps_result (query, ret); + ret = TRUE; + break; + } case GST_QUERY_ALLOCATION: { GstQuery *decide_query = NULL; From 33584480b5e2f75db6f6f1faf55fbcac67ce53e2 Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Fri, 24 Oct 2014 00:35:22 +1100 Subject: [PATCH 066/381] glmixer: override the caps query in order to 'convert' capsfeatures Otherwise, it is only possible for the sink pads and the src pads to have the exact same caps features. We can convert from any feature to another feature so support that. --- ext/gl/gstglmixer.c | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/ext/gl/gstglmixer.c b/ext/gl/gstglmixer.c index da6ba67ed0..a80d2b726c 100644 --- a/ext/gl/gstglmixer.c +++ b/ext/gl/gstglmixer.c @@ -287,6 +287,37 @@ gst_gl_mixer_pad_sink_acceptcaps (GstPad * pad, GstGLMixer * mix, return ret; } +static GstCaps * +gst_gl_mixer_pad_sink_getcaps (GstPad * pad, GstGLMixer * mix, GstCaps * filter) +{ + GstCaps *srccaps; + GstCaps *template_caps; + GstCaps *filtered_caps; + GstCaps *returned_caps; + gboolean had_current_caps = TRUE; + + template_caps = gst_pad_get_pad_template_caps (pad); + + srccaps = gst_pad_get_current_caps (pad); + if (srccaps == NULL) { + had_current_caps = FALSE; + srccaps = template_caps; + } + + filtered_caps = srccaps; + if (filter) + filtered_caps = gst_caps_intersect (srccaps, filter); + returned_caps = gst_caps_intersect (filtered_caps, template_caps); + + gst_caps_unref (srccaps); + if (filter) + gst_caps_unref (filtered_caps); + if (had_current_caps) + gst_caps_unref (template_caps); + + return returned_caps; +} + static gboolean gst_gl_mixer_sink_query (GstAggregator * agg, GstAggregatorPad * bpad, GstQuery * query) @@ -297,6 +328,17 @@ gst_gl_mixer_sink_query (GstAggregator * agg, GstAggregatorPad * bpad, GST_TRACE ("QUERY %" GST_PTR_FORMAT, query); switch (GST_QUERY_TYPE (query)) { + case GST_QUERY_CAPS: + { + GstCaps *filter, *caps; + + gst_query_parse_caps (query, &filter); + caps = gst_gl_mixer_pad_sink_getcaps (GST_PAD (bpad), mix, filter); + gst_query_set_caps_result (query, caps); + gst_caps_unref (caps); + ret = TRUE; + break; + } case GST_QUERY_ACCEPT_CAPS: { GstCaps *caps; From b151f564c2142a473510e2b247f7178539641dce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wang=20Xin-yu=20=28=E7=8E=8B=E6=98=95=E5=AE=87=29?= Date: Wed, 29 Oct 2014 08:27:57 +0800 Subject: [PATCH 067/381] glmixer:fix incorrect parameter passed to handle_set_context --- ext/gl/gstglmixer.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ext/gl/gstglmixer.c b/ext/gl/gstglmixer.c index a80d2b726c..1c6cfe9c3b 100644 --- a/ext/gl/gstglmixer.c +++ b/ext/gl/gstglmixer.c @@ -552,7 +552,8 @@ gst_gl_mixer_set_context (GstElement * element, GstContext * context) { GstGLMixer *mix = GST_GL_MIXER (element); - gst_gl_handle_set_context (element, context, &mix->display, &mix->context); + gst_gl_handle_set_context (element, context, &mix->display, + &mix->other_context); } static gboolean From eb11906f01ef4ec8111374abe5c43c40d3cfbd6e Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Thu, 30 Oct 2014 23:08:00 +1100 Subject: [PATCH 068/381] glmixer: don't get the current caps from GstVideoInfo for the srcpad It's missing the caps features needed. --- ext/gl/gstglmixer.c | 63 +++++++++++++++++---------------------------- 1 file changed, 24 insertions(+), 39 deletions(-) diff --git a/ext/gl/gstglmixer.c b/ext/gl/gstglmixer.c index 1c6cfe9c3b..f95a955fd7 100644 --- a/ext/gl/gstglmixer.c +++ b/ext/gl/gstglmixer.c @@ -590,45 +590,6 @@ gst_gl_mixer_src_activate_mode (GstAggregator * aggregator, GstPadMode mode, return result; } -static gboolean -gst_gl_mixer_query_caps (GstPad * pad, GstAggregator * agg, GstQuery * query) -{ - GstCaps *filter, *caps; - GstStructure *s; - gint n; - - GstVideoAggregator *vagg = GST_VIDEO_AGGREGATOR (agg); - - gst_query_parse_caps (query, &filter); - - if (GST_VIDEO_INFO_FORMAT (&vagg->info) != GST_VIDEO_FORMAT_UNKNOWN) { - caps = gst_video_info_to_caps (&vagg->info); - } else { - caps = gst_pad_get_pad_template_caps (agg->srcpad); - } - - caps = gst_caps_make_writable (caps); - - n = gst_caps_get_size (caps) - 1; - for (; n >= 0; n--) { - s = gst_caps_get_structure (caps, n); - gst_structure_set (s, "width", GST_TYPE_INT_RANGE, 1, G_MAXINT, - "height", GST_TYPE_INT_RANGE, 1, G_MAXINT, NULL); - if (GST_VIDEO_INFO_FPS_D (&vagg->info) != 0) { - gst_structure_set (s, - "framerate", GST_TYPE_FRACTION_RANGE, 0, 1, G_MAXINT, 1, NULL); - } - } - - if (filter) - caps = gst_caps_intersect_full (filter, caps, GST_CAPS_INTERSECT_FIRST); - - gst_query_set_caps_result (query, caps); - gst_caps_unref (caps); - - return TRUE; -} - static GstCaps * gst_gl_mixer_set_caps_features (const GstCaps * caps, const gchar * feature_name) @@ -721,6 +682,30 @@ gst_gl_mixer_update_caps (GstGLMixer * mix, GstCaps * caps) return result; } +static gboolean +gst_gl_mixer_query_caps (GstPad * pad, GstAggregator * agg, GstQuery * query) +{ + GstCaps *filter, *current_caps, *retcaps; + + gst_query_parse_caps (query, &filter); + + current_caps = gst_pad_get_current_caps (pad); + if (current_caps == NULL) + current_caps = gst_pad_get_pad_template_caps (agg->srcpad); + + retcaps = gst_gl_mixer_caps_remove_format_info (current_caps); + gst_caps_unref (current_caps); + + if (filter) + retcaps = + gst_caps_intersect_full (filter, retcaps, GST_CAPS_INTERSECT_FIRST); + + gst_query_set_caps_result (query, retcaps); + gst_caps_unref (retcaps); + + return TRUE; +} + static gboolean gst_gl_mixer_src_query (GstAggregator * agg, GstQuery * query) { From 2bf4a941791f738212e28a9b039d0ef845f11cf7 Mon Sep 17 00:00:00 2001 From: Luis de Bethencourt Date: Thu, 30 Oct 2014 14:49:05 +0000 Subject: [PATCH 069/381] videoaggregator: remove storage of never used values These two values are stored just before the function returns and they go out of scope. --- gst-libs/gst/video/gstvideoaggregator.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index e24bb212e4..b339a76b3e 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -1049,7 +1049,6 @@ prepare_frames (GstVideoAggregator * vagg, GstVideoAggregatorPad * pad) g_slice_free (GstVideoFrame, frame); } else { converted_frame = frame; - converted_buf = pad->buffer; } pad->aggregated_frame = converted_frame; @@ -1448,7 +1447,6 @@ gst_videoaggregator_sink_clip (GstAggregator * agg, /* Convert to the output segment rate */ if (ABS (agg->segment.rate) != 1.0) { - start_time *= ABS (agg->segment.rate); end_time *= ABS (agg->segment.rate); } From 83c34503af44a50d506d0b8eff1f2fcf46838664 Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Fri, 31 Oct 2014 12:52:07 +1100 Subject: [PATCH 070/381] glmixer: advertise support for changing input caps mid-stream https://bugzilla.gnome.org/show_bug.cgi?id=739334 --- ext/gl/gstglmixer.c | 189 ++++++++++++++++++++++---------------------- 1 file changed, 96 insertions(+), 93 deletions(-) diff --git a/ext/gl/gstglmixer.c b/ext/gl/gstglmixer.c index f95a955fd7..9b1dcb5ed8 100644 --- a/ext/gl/gstglmixer.c +++ b/ext/gl/gstglmixer.c @@ -287,6 +287,98 @@ gst_gl_mixer_pad_sink_acceptcaps (GstPad * pad, GstGLMixer * mix, return ret; } +static GstCaps * +gst_gl_mixer_set_caps_features (const GstCaps * caps, + const gchar * feature_name) +{ + GstCaps *tmp = gst_caps_copy (caps); + guint n = gst_caps_get_size (tmp); + guint i = 0; + + for (i = 0; i < n; i++) { + GstCapsFeatures *features = gst_caps_get_features (tmp, i); + if (features) { + guint n_f = gst_caps_features_get_size (features); + guint j = 0; + for (j = 0; j < n_f; j++) { + gst_caps_features_remove_id (features, + gst_caps_features_get_nth_id (features, j)); + } + } + + gst_caps_features_add (features, feature_name); + gst_caps_set_simple (tmp, "format", G_TYPE_STRING, "RGBA", NULL); + } + + return tmp; +} + +/* copies the given caps */ +static GstCaps * +gst_gl_mixer_caps_remove_format_info (GstCaps * caps) +{ + GstStructure *st; + GstCapsFeatures *f; + gint i, n; + GstCaps *res; + + res = gst_caps_new_empty (); + + n = gst_caps_get_size (caps); + for (i = 0; i < n; i++) { + st = gst_caps_get_structure (caps, i); + f = gst_caps_get_features (caps, i); + + /* If this is already expressed by the existing caps + * skip this structure */ + if (i > 0 && gst_caps_is_subset_structure_full (res, st, f)) + continue; + + st = gst_structure_copy (st); + /* Only remove format info for the cases when we can actually convert */ + if (!gst_caps_features_is_any (f) + && gst_caps_features_is_equal (f, + GST_CAPS_FEATURES_MEMORY_SYSTEM_MEMORY)) + gst_structure_remove_fields (st, "format", "colorimetry", "chroma-site", + "width", "height", NULL); + + gst_caps_append_structure_full (res, st, gst_caps_features_copy (f)); + } + + return res; +} + +GstCaps * +gst_gl_mixer_update_caps (GstGLMixer * mix, GstCaps * caps) +{ + GstCaps *result = NULL; + GstCaps *glcaps = gst_gl_mixer_set_caps_features (caps, + GST_CAPS_FEATURE_MEMORY_GL_MEMORY); +#if GST_GL_HAVE_PLATFORM_EGL + GstCaps *eglcaps = gst_gl_mixer_set_caps_features (caps, + GST_CAPS_FEATURE_MEMORY_EGL_IMAGE); +#endif + GstCaps *uploadcaps = gst_gl_mixer_set_caps_features (caps, + GST_CAPS_FEATURE_META_GST_VIDEO_GL_TEXTURE_UPLOAD_META); + GstCaps *raw_caps = + gst_caps_from_string (GST_VIDEO_CAPS_MAKE (GST_GL_COLOR_CONVERT_FORMATS)); + + result = gst_caps_new_empty (); + + result = gst_caps_merge (result, glcaps); +#if GST_GL_HAVE_PLATFORM_EGL + result = gst_caps_merge (result, eglcaps); +#endif + result = gst_caps_merge (result, uploadcaps); + result = gst_caps_merge (result, raw_caps); + + result = gst_caps_merge (result, gst_gl_mixer_caps_remove_format_info (caps)); + + GST_DEBUG_OBJECT (mix, "returning %" GST_PTR_FORMAT, result); + + return result; +} + static GstCaps * gst_gl_mixer_pad_sink_getcaps (GstPad * pad, GstGLMixer * mix, GstCaps * filter) { @@ -302,6 +394,8 @@ gst_gl_mixer_pad_sink_getcaps (GstPad * pad, GstGLMixer * mix, GstCaps * filter) if (srccaps == NULL) { had_current_caps = FALSE; srccaps = template_caps; + } else { + srccaps = gst_caps_merge (srccaps, gst_gl_mixer_update_caps (mix, srccaps)); } filtered_caps = srccaps; @@ -309,12 +403,13 @@ gst_gl_mixer_pad_sink_getcaps (GstPad * pad, GstGLMixer * mix, GstCaps * filter) filtered_caps = gst_caps_intersect (srccaps, filter); returned_caps = gst_caps_intersect (filtered_caps, template_caps); - gst_caps_unref (srccaps); if (filter) gst_caps_unref (filtered_caps); if (had_current_caps) gst_caps_unref (template_caps); + GST_DEBUG_OBJECT (pad, "returning %" GST_PTR_FORMAT, returned_caps); + return returned_caps; } @@ -590,98 +685,6 @@ gst_gl_mixer_src_activate_mode (GstAggregator * aggregator, GstPadMode mode, return result; } -static GstCaps * -gst_gl_mixer_set_caps_features (const GstCaps * caps, - const gchar * feature_name) -{ - GstCaps *tmp = gst_caps_copy (caps); - guint n = gst_caps_get_size (tmp); - guint i = 0; - - for (i = 0; i < n; i++) { - GstCapsFeatures *features = gst_caps_get_features (tmp, i); - if (features) { - guint n_f = gst_caps_features_get_size (features); - guint j = 0; - for (j = 0; j < n_f; j++) { - gst_caps_features_remove_id (features, - gst_caps_features_get_nth_id (features, j)); - } - } - - gst_caps_features_add (features, feature_name); - gst_caps_set_simple (tmp, "format", G_TYPE_STRING, "RGBA", NULL); - } - - return tmp; -} - -/* copies the given caps */ -static GstCaps * -gst_gl_mixer_caps_remove_format_info (GstCaps * caps) -{ - GstStructure *st; - GstCapsFeatures *f; - gint i, n; - GstCaps *res; - - res = gst_caps_new_empty (); - - n = gst_caps_get_size (caps); - for (i = 0; i < n; i++) { - st = gst_caps_get_structure (caps, i); - f = gst_caps_get_features (caps, i); - - /* If this is already expressed by the existing caps - * skip this structure */ - if (i > 0 && gst_caps_is_subset_structure_full (res, st, f)) - continue; - - st = gst_structure_copy (st); - /* Only remove format info for the cases when we can actually convert */ - if (!gst_caps_features_is_any (f) - && gst_caps_features_is_equal (f, - GST_CAPS_FEATURES_MEMORY_SYSTEM_MEMORY)) - gst_structure_remove_fields (st, "format", "colorimetry", "chroma-site", - "width", "height", NULL); - - gst_caps_append_structure_full (res, st, gst_caps_features_copy (f)); - } - - return res; -} - -GstCaps * -gst_gl_mixer_update_caps (GstGLMixer * mix, GstCaps * caps) -{ - GstCaps *result = NULL; - GstCaps *glcaps = gst_gl_mixer_set_caps_features (caps, - GST_CAPS_FEATURE_MEMORY_GL_MEMORY); -#if GST_GL_HAVE_PLATFORM_EGL - GstCaps *eglcaps = gst_gl_mixer_set_caps_features (caps, - GST_CAPS_FEATURE_MEMORY_EGL_IMAGE); -#endif - GstCaps *uploadcaps = gst_gl_mixer_set_caps_features (caps, - GST_CAPS_FEATURE_META_GST_VIDEO_GL_TEXTURE_UPLOAD_META); - GstCaps *raw_caps = - gst_caps_from_string (GST_VIDEO_CAPS_MAKE (GST_GL_COLOR_CONVERT_FORMATS)); - - result = gst_caps_new_empty (); - - result = gst_caps_merge (result, glcaps); -#if GST_GL_HAVE_PLATFORM_EGL - result = gst_caps_merge (result, eglcaps); -#endif - result = gst_caps_merge (result, uploadcaps); - result = gst_caps_merge (result, raw_caps); - - result = gst_caps_merge (result, gst_gl_mixer_caps_remove_format_info (caps)); - - GST_DEBUG_OBJECT (mix, "returning %" GST_PTR_FORMAT, result); - - return result; -} - static gboolean gst_gl_mixer_query_caps (GstPad * pad, GstAggregator * agg, GstQuery * query) { From 22825010c36e037fa851e97e1b77f6649113bae3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Mon, 3 Nov 2014 16:13:23 +0100 Subject: [PATCH 071/381] videoaggregator: Swap source/destination parameters of gst_video_converter_frame() --- gst-libs/gst/video/gstvideoaggregator.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index b339a76b3e..115bde2496 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -1043,7 +1043,7 @@ prepare_frames (GstVideoAggregator * vagg, GstVideoAggregatorPad * pad) return FALSE; } - gst_video_converter_frame (pad->priv->convert, converted_frame, frame); + gst_video_converter_frame (pad->priv->convert, frame, converted_frame); pad->converted_buffer = converted_buf; gst_video_frame_unmap (frame); g_slice_free (GstVideoFrame, frame); From f9c8268247dffa4ee380b10e05cac104ff17b8d5 Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Tue, 11 Nov 2014 14:23:55 +1100 Subject: [PATCH 072/381] gl: remove the width/height fields from the caps to support frame resizing It was previously only occuring with sysmem caps features https://bugzilla.gnome.org/show_bug.cgi?id=739334 --- ext/gl/gstglmixer.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ext/gl/gstglmixer.c b/ext/gl/gstglmixer.c index 9b1dcb5ed8..b6b88046d6 100644 --- a/ext/gl/gstglmixer.c +++ b/ext/gl/gstglmixer.c @@ -340,7 +340,8 @@ gst_gl_mixer_caps_remove_format_info (GstCaps * caps) && gst_caps_features_is_equal (f, GST_CAPS_FEATURES_MEMORY_SYSTEM_MEMORY)) gst_structure_remove_fields (st, "format", "colorimetry", "chroma-site", - "width", "height", NULL); + NULL); + gst_structure_remove_fields (st, "width", "height", NULL); gst_caps_append_structure_full (res, st, gst_caps_features_copy (f)); } From d28854a625514c148ead2c18186f27d7c5b28274 Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Mon, 17 Nov 2014 18:50:04 +1100 Subject: [PATCH 073/381] glmixer: add read-only context property --- ext/gl/gstglmixer.c | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/ext/gl/gstglmixer.c b/ext/gl/gstglmixer.c index b6b88046d6..a10d10d37c 100644 --- a/ext/gl/gstglmixer.c +++ b/ext/gl/gstglmixer.c @@ -501,7 +501,8 @@ enum enum { - PROP_0 + PROP_0, + PROP_CONTEXT }; static GstStaticPadTemplate src_factory = GST_STATIC_PAD_TEMPLATE ("src", @@ -596,6 +597,11 @@ gst_gl_mixer_class_init (GstGLMixerClass * klass) videoaggregator_class->get_output_buffer = gst_gl_mixer_get_output_buffer; videoaggregator_class->negotiated_caps = _negotiated_caps; + g_object_class_install_property (gobject_class, PROP_CONTEXT, + g_param_spec_object ("context", + "OpenGL context", + "Get OpenGL context", + GST_GL_TYPE_CONTEXT, G_PARAM_READABLE | G_PARAM_STATIC_STRINGS)); /* Register the pad class */ g_type_class_ref (GST_TYPE_GL_MIXER_PAD); @@ -1176,7 +1182,12 @@ static void gst_gl_mixer_get_property (GObject * object, guint prop_id, GValue * value, GParamSpec * pspec) { + GstGLMixer *mixer = GST_GL_MIXER (object); + switch (prop_id) { + case PROP_CONTEXT: + g_value_set_object (value, mixer->context); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); break; From 0ec9bf6140b917118f7841bf4029b13b37a80561 Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Mon, 17 Nov 2014 14:05:01 +1100 Subject: [PATCH 074/381] videoaggregator: fix up QoS handling for live sources Only attempt adaptive drop when we are not live https://bugzilla.gnome.org/show_bug.cgi?id=739996 --- gst-libs/gst/video/gstvideoaggregator.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index 115bde2496..83ac15beb0 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -749,15 +749,19 @@ static void gst_videoaggregator_update_qos (GstVideoAggregator * vagg, gdouble proportion, GstClockTimeDiff diff, GstClockTime timestamp) { + gboolean live; + GST_DEBUG_OBJECT (vagg, "Updating QoS: proportion %lf, diff %s%" GST_TIME_FORMAT ", timestamp %" GST_TIME_FORMAT, proportion, (diff < 0) ? "-" : "", GST_TIME_ARGS (ABS (diff)), GST_TIME_ARGS (timestamp)); GST_OBJECT_LOCK (vagg); + gst_aggregator_get_latency (GST_AGGREGATOR (vagg), &live, NULL, NULL); + vagg->priv->proportion = proportion; if (G_LIKELY (timestamp != GST_CLOCK_TIME_NONE)) { - if (G_UNLIKELY (diff > 0)) + if (!live && G_UNLIKELY (diff > 0)) vagg->priv->earliest_time = timestamp + 2 * diff + gst_util_uint64_scale_int_round (GST_SECOND, GST_VIDEO_INFO_FPS_D (&vagg->info), From f2364cb39840241f4415c9d773f90691626600ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Wed, 19 Nov 2014 17:02:40 +0100 Subject: [PATCH 075/381] videoaggregator: Don't output 0-duration buffers at the segment end https://bugzilla.gnome.org/show_bug.cgi?id=740376 --- gst-libs/gst/video/gstvideoaggregator.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index 83ac15beb0..720e702d33 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -1190,9 +1190,13 @@ gst_videoaggregator_aggregate (GstAggregator * agg) if (agg->segment.stop != -1) output_end_time = MIN (output_end_time, agg->segment.stop); - res = - gst_videoaggregator_fill_queues (vagg, output_start_time, - output_end_time); + if (output_end_time == output_start_time) { + res = GST_FLOW_EOS; + } else { + res = + gst_videoaggregator_fill_queues (vagg, output_start_time, + output_end_time); + } if (res == GST_FLOW_NEEDS_DATA) { GST_DEBUG_OBJECT (vagg, "Need more data for decisions"); From c337aab9915e13dd69e9dff97c4b3582324d20fd Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Wed, 5 Nov 2014 20:18:06 +1100 Subject: [PATCH 076/381] glupload: rearchitecture for non GLMemory inputs/outputs Allows other memory types to be implemented/returned/used by the caller. --- ext/gl/gstglmixer.c | 40 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 36 insertions(+), 4 deletions(-) diff --git a/ext/gl/gstglmixer.c b/ext/gl/gstglmixer.c index a10d10d37c..cd77aaf90c 100644 --- a/ext/gl/gstglmixer.c +++ b/ext/gl/gstglmixer.c @@ -1064,23 +1064,55 @@ gst_gl_mixer_process_textures (GstGLMixer * mix, GstBuffer * outbuf) walk = g_list_next (walk); if (vaggpad->buffer != NULL) { - guint in_tex; + GstBuffer *gl_buf; + GstCaps *gl_caps; + GstCapsFeatures *gl_features; + GstVideoInfo gl_info; + GstVideoFrame gl_frame; + + gst_video_info_set_format (&gl_info, + GST_VIDEO_FORMAT_RGBA, + GST_VIDEO_INFO_WIDTH (&vaggpad->info), + GST_VIDEO_INFO_HEIGHT (&vaggpad->info)); + gl_features = + gst_caps_features_from_string (GST_CAPS_FEATURE_MEMORY_GL_MEMORY); + + gl_caps = gst_video_info_to_caps (&gl_info); + gst_caps_set_features (gl_caps, 0, gl_features); if (!pad->upload) { + GstCaps *in_caps = gst_pad_get_current_caps (GST_PAD (pad)); + pad->upload = gst_gl_upload_new (mix->context); - gst_gl_upload_set_format (pad->upload, &vaggpad->info); + gst_gl_upload_set_caps (pad->upload, in_caps, gl_caps); + + gst_caps_unref (in_caps); } if (!gst_gl_upload_perform_with_buffer (pad->upload, - vaggpad->buffer, &in_tex, NULL)) { + vaggpad->buffer, &gl_buf)) { ++array_index; pad->mapped = FALSE; + gst_caps_unref (gl_caps); + continue; + } + + if (!gst_video_frame_map (&gl_frame, &gl_info, gl_buf, + GST_MAP_READ | GST_MAP_GL)) { + ++array_index; + pad->mapped = FALSE; + gst_buffer_unref (gl_buf); + gst_caps_unref (gl_caps); continue; } pad->mapped = TRUE; - frame->texture = in_tex; + frame->texture = *(guint *) gl_frame.data[0]; + + gst_caps_unref (gl_caps); + gst_video_frame_unmap (&gl_frame); + gst_buffer_unref (gl_buf); } ++array_index; } From 170def9d430c0affd68a0d0b84f188c0d78f1430 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Tue, 25 Nov 2014 18:53:55 +0100 Subject: [PATCH 077/381] compositor: GstVideoMeta is supported just fine, tell upstream about that Also provide a GstVideoBufferPool to upstream just in case. --- gst/compositor/compositor.c | 43 +++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/gst/compositor/compositor.c b/gst/compositor/compositor.c index 8b68950c23..6a7fa003c5 100644 --- a/gst/compositor/compositor.c +++ b/gst/compositor/compositor.c @@ -549,6 +549,48 @@ gst_compositor_aggregate_frames (GstVideoAggregator * vagg, GstBuffer * outbuf) return GST_FLOW_OK; } +static gboolean +_sink_query (GstAggregator * agg, GstAggregatorPad * bpad, GstQuery * query) +{ + switch (GST_QUERY_TYPE (query)) { + case GST_QUERY_ALLOCATION:{ + GstCaps *caps; + GstVideoInfo info; + GstBufferPool *pool; + guint size; + GstStructure *structure; + + gst_query_parse_allocation (query, &caps, NULL); + + if (caps == NULL) + return FALSE; + + if (!gst_video_info_from_caps (&info, caps)) + return FALSE; + + size = GST_VIDEO_INFO_SIZE (&info); + + pool = gst_video_buffer_pool_new (); + + structure = gst_buffer_pool_get_config (pool); + gst_buffer_pool_config_set_params (structure, caps, size, 0, 0); + + if (!gst_buffer_pool_set_config (pool, structure)) { + gst_object_unref (pool); + return FALSE; + } + + gst_query_add_allocation_pool (query, pool, size, 0, 0); + gst_object_unref (pool); + gst_query_add_allocation_meta (query, GST_VIDEO_META_API_TYPE, NULL); + + return TRUE; + } + default: + return GST_AGGREGATOR_CLASS (parent_class)->sink_query (agg, bpad, query); + } +} + /* GObject boilerplate */ static void gst_compositor_class_init (GstCompositorClass * klass) @@ -563,6 +605,7 @@ gst_compositor_class_init (GstCompositorClass * klass) gobject_class->set_property = gst_compositor_set_property; agg_class->sinkpads_type = GST_TYPE_COMPOSITOR_PAD; + agg_class->sink_query = _sink_query; videoaggregator_class->update_caps = _update_caps; videoaggregator_class->aggregate_frames = gst_compositor_aggregate_frames; From dbadc6c8a52674b9309068822e917798b98c4ed5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Wed, 26 Nov 2014 12:35:52 +0100 Subject: [PATCH 078/381] compositor: Remove unused zorder pad property It's handled in videoaggregator already. --- gst/compositor/compositor.c | 5 ----- gst/compositor/compositorpad.h | 1 - 2 files changed, 6 deletions(-) diff --git a/gst/compositor/compositor.c b/gst/compositor/compositor.c index 6a7fa003c5..b7848b47a7 100644 --- a/gst/compositor/compositor.c +++ b/gst/compositor/compositor.c @@ -112,14 +112,12 @@ static GstStaticPadTemplate sink_factory = GST_STATIC_PAD_TEMPLATE ("sink_%u", GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE (FORMATS)) ); -#define DEFAULT_PAD_ZORDER 0 #define DEFAULT_PAD_XPOS 0 #define DEFAULT_PAD_YPOS 0 #define DEFAULT_PAD_ALPHA 1.0 enum { PROP_PAD_0, - PROP_PAD_ZORDER, PROP_PAD_XPOS, PROP_PAD_YPOS, PROP_PAD_ALPHA @@ -135,9 +133,6 @@ gst_compositor_pad_get_property (GObject * object, guint prop_id, GstCompositorPad *pad = GST_COMPOSITOR_PAD (object); switch (prop_id) { - case PROP_PAD_ZORDER: - g_value_set_uint (value, pad->zorder); - break; case PROP_PAD_XPOS: g_value_set_int (value, pad->xpos); break; diff --git a/gst/compositor/compositorpad.h b/gst/compositor/compositorpad.h index 0ba580e5b1..a1c83f8d53 100644 --- a/gst/compositor/compositorpad.h +++ b/gst/compositor/compositorpad.h @@ -50,7 +50,6 @@ struct _GstCompositorPad /* properties */ gint xpos, ypos; - guint zorder; gdouble alpha; }; From fc593bd2ca983656e2c318319d2ea33562473f72 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Wed, 26 Nov 2014 13:06:21 +0100 Subject: [PATCH 079/381] videoaggregator: Also sync pad properties to the controller if conversion is disabled --- gst-libs/gst/video/gstvideoaggregator.c | 91 +++++++++++++------------ 1 file changed, 46 insertions(+), 45 deletions(-) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index 720e702d33..3dfa1b68a6 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -990,17 +990,12 @@ static gboolean prepare_frames (GstVideoAggregator * vagg, GstVideoAggregatorPad * pad) { GstAggregatorPad *bpad = GST_AGGREGATOR_PAD (pad); - - static GstAllocationParams params = { 0, 15, 0, 0, }; + GstVideoAggregatorClass *vagg_klass = GST_VIDEO_AGGREGATOR_GET_CLASS (vagg); if (pad->buffer != NULL) { - guint outsize; GstClockTime timestamp; gint64 stream_time; GstSegment *seg; - GstVideoFrame *converted_frame; - GstBuffer *converted_buf = NULL; - GstVideoFrame *frame = g_slice_new0 (GstVideoFrame); seg = &bpad->segment; @@ -1012,50 +1007,57 @@ prepare_frames (GstVideoAggregator * vagg, GstVideoAggregatorPad * pad) if (GST_CLOCK_TIME_IS_VALID (stream_time)) gst_object_sync_values (GST_OBJECT (pad), stream_time); + if (!vagg_klass->disable_frame_conversion) { + guint outsize; + GstVideoFrame *converted_frame; + GstBuffer *converted_buf = NULL; + GstVideoFrame *frame = g_slice_new0 (GstVideoFrame); + static GstAllocationParams params = { 0, 15, 0, 0, }; - if (!gst_video_frame_map (frame, &pad->buffer_vinfo, pad->buffer, - GST_MAP_READ)) { - GST_WARNING_OBJECT (vagg, "Could not map input buffer"); - } - - if (pad->priv->convert) { - gint converted_size; - - converted_frame = g_slice_new0 (GstVideoFrame); - - /* We wait until here to set the conversion infos, in case vagg->info changed */ - if (pad->need_conversion_update) { - pad->conversion_info = vagg->info; - gst_video_info_set_format (&(pad->conversion_info), - GST_VIDEO_INFO_FORMAT (&vagg->info), pad->info.width, - pad->info.height); - pad->need_conversion_update = FALSE; + if (!gst_video_frame_map (frame, &pad->buffer_vinfo, pad->buffer, + GST_MAP_READ)) { + GST_WARNING_OBJECT (vagg, "Could not map input buffer"); } - converted_size = pad->conversion_info.size; - outsize = GST_VIDEO_INFO_SIZE (&vagg->info); - converted_size = converted_size > outsize ? converted_size : outsize; - converted_buf = gst_buffer_new_allocate (NULL, converted_size, ¶ms); + if (pad->priv->convert) { + gint converted_size; - if (!gst_video_frame_map (converted_frame, &(pad->conversion_info), - converted_buf, GST_MAP_READWRITE)) { - GST_WARNING_OBJECT (vagg, "Could not map converted frame"); + converted_frame = g_slice_new0 (GstVideoFrame); - g_slice_free (GstVideoFrame, converted_frame); + /* We wait until here to set the conversion infos, in case vagg->info changed */ + if (pad->need_conversion_update) { + pad->conversion_info = vagg->info; + gst_video_info_set_format (&(pad->conversion_info), + GST_VIDEO_INFO_FORMAT (&vagg->info), pad->info.width, + pad->info.height); + pad->need_conversion_update = FALSE; + } + + converted_size = pad->conversion_info.size; + outsize = GST_VIDEO_INFO_SIZE (&vagg->info); + converted_size = converted_size > outsize ? converted_size : outsize; + converted_buf = gst_buffer_new_allocate (NULL, converted_size, ¶ms); + + if (!gst_video_frame_map (converted_frame, &(pad->conversion_info), + converted_buf, GST_MAP_READWRITE)) { + GST_WARNING_OBJECT (vagg, "Could not map converted frame"); + + g_slice_free (GstVideoFrame, converted_frame); + gst_video_frame_unmap (frame); + g_slice_free (GstVideoFrame, frame); + return FALSE; + } + + gst_video_converter_frame (pad->priv->convert, frame, converted_frame); + pad->converted_buffer = converted_buf; gst_video_frame_unmap (frame); g_slice_free (GstVideoFrame, frame); - return FALSE; + } else { + converted_frame = frame; } - gst_video_converter_frame (pad->priv->convert, frame, converted_frame); - pad->converted_buffer = converted_buf; - gst_video_frame_unmap (frame); - g_slice_free (GstVideoFrame, frame); - } else { - converted_frame = frame; + pad->aggregated_frame = converted_frame; } - - pad->aggregated_frame = converted_frame; } return TRUE; @@ -1099,11 +1101,10 @@ gst_videoaggregator_do_aggregate (GstVideoAggregator * vagg, GST_BUFFER_TIMESTAMP (*outbuf) = output_start_time; GST_BUFFER_DURATION (*outbuf) = output_end_time - output_start_time; - if (vagg_klass->disable_frame_conversion == FALSE) { - /* Here we convert all the frames the subclass will have to aggregate */ - gst_aggregator_iterate_sinkpads (GST_AGGREGATOR (vagg), - (GstAggregatorPadForeachFunc) prepare_frames, NULL); - } + /* Here we convert all the frames the subclass will have to aggregate + * and also sync pad properties to the stream time */ + gst_aggregator_iterate_sinkpads (GST_AGGREGATOR (vagg), + (GstAggregatorPadForeachFunc) prepare_frames, NULL); ret = vagg_klass->aggregate_frames (vagg, *outbuf); From a806b401352b34a832937dea7398811ce6458135 Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Fri, 31 Oct 2014 11:01:47 +0100 Subject: [PATCH 080/381] videoaggregator: Let a full renegotiation happen after removing the last pad With the current code, we will end up setting the preferred downstream format as the srcpad format, and it might not be accepted by the next sinkpad to be added. We should instead let the next sinkpad reconfigure everything. --- gst-libs/gst/video/gstvideoaggregator.c | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index 3dfa1b68a6..6a0fa558b3 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -1609,13 +1609,22 @@ gst_videoaggregator_release_pad (GstElement * element, GstPad * pad) { GstVideoAggregator *vagg = NULL; GstVideoAggregatorPad *vaggpad; - gboolean update_caps; + gboolean update_caps, last_pad; vagg = GST_VIDEO_AGGREGATOR (element); vaggpad = GST_VIDEO_AGGREGATOR_PAD (pad); GST_VIDEO_AGGREGATOR_LOCK (vagg); - gst_videoaggregator_update_converters (vagg); + + GST_OBJECT_LOCK (vagg); + last_pad = (GST_ELEMENT (vagg)->numsinkpads - 1 == 0); + GST_OBJECT_UNLOCK (vagg); + + if (!last_pad) + gst_videoaggregator_update_converters (vagg); + else + gst_videoaggregator_reset (vagg); + gst_buffer_replace (&vaggpad->buffer, NULL); update_caps = GST_VIDEO_INFO_FORMAT (&vagg->info) != GST_VIDEO_FORMAT_UNKNOWN; From 8cc214fdb80f06b2a80a59192c1f256561c2b4ef Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Wed, 26 Nov 2014 18:24:05 +0100 Subject: [PATCH 081/381] videoaggregator: Expose vmethods to set converters and prepare/clean frames This gives more flexibility to the subclasses and permits to remove the GstVideoAggregatorClass->disable_frame_conversion ugly API. WARNING: This breaks the API as it removes the disable_frame_conversion field API: + GstVideoAggregatorClass->find_best_format + GstVideoAggregatorPadClass->set_format + GstVideoAggregatorPadClass->prepare_frame + GstVideoAggregatorPadClass->clean_frame - GstVideoAggregatorClass->disable_frame_conversion https://bugzilla.gnome.org/show_bug.cgi?id=740768 --- ext/gl/gstglmixer.c | 8 +- gst-libs/gst/video/gstvideoaggregator.c | 295 ++++++++++++++---------- gst-libs/gst/video/gstvideoaggregator.h | 15 +- 3 files changed, 184 insertions(+), 134 deletions(-) diff --git a/ext/gl/gstglmixer.c b/ext/gl/gstglmixer.c index cd77aaf90c..bd63c75a1f 100644 --- a/ext/gl/gstglmixer.c +++ b/ext/gl/gstglmixer.c @@ -87,11 +87,17 @@ static void gst_gl_mixer_pad_class_init (GstGLMixerPadClass * klass) { GObjectClass *gobject_class = (GObjectClass *) klass; + GstVideoAggregatorPadClass *vaggpad_class = + (GstVideoAggregatorPadClass *) klass; gobject_class->set_property = gst_gl_mixer_pad_set_property; gobject_class->get_property = gst_gl_mixer_pad_get_property; gobject_class->finalize = gst_gl_mixer_pad_finalize; + + vaggpad_class->set_info = NULL; + vaggpad_class->prepare_frame = NULL; + vaggpad_class->clean_frame = NULL; } static void @@ -592,10 +598,10 @@ gst_gl_mixer_class_init (GstGLMixerClass * klass) agg_class->stop = gst_gl_mixer_stop; agg_class->start = gst_gl_mixer_start; - videoaggregator_class->disable_frame_conversion = TRUE; videoaggregator_class->aggregate_frames = gst_gl_mixer_aggregate_frames; videoaggregator_class->get_output_buffer = gst_gl_mixer_get_output_buffer; videoaggregator_class->negotiated_caps = _negotiated_caps; + videoaggregator_class->find_best_format = NULL; g_object_class_install_property (gobject_class, PROP_CONTEXT, g_param_spec_object ("context", diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index 6a0fa558b3..9e9bb3a345 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -131,6 +131,63 @@ _flush_pad (GstAggregatorPad * aggpad, GstAggregator * aggregator) return TRUE; } +static gboolean +gst_video_aggregator_pad_set_info (GstVideoAggregatorPad * pad, + GstVideoAggregator * vagg G_GNUC_UNUSED, + GstVideoInfo * current_info, GstVideoInfo * wanted_info) +{ + gchar *colorimetry, *best_colorimetry; + const gchar *chroma, *best_chroma; + + if (!current_info->finfo) + return TRUE; + + if (GST_VIDEO_INFO_FORMAT (current_info) == GST_VIDEO_FORMAT_UNKNOWN) + return TRUE; + + if (pad->priv->convert) + gst_video_converter_free (pad->priv->convert); + + pad->priv->convert = NULL; + + colorimetry = gst_video_colorimetry_to_string (&(current_info->colorimetry)); + chroma = gst_video_chroma_to_string (current_info->chroma_site); + + best_colorimetry = + gst_video_colorimetry_to_string (&(wanted_info->colorimetry)); + best_chroma = gst_video_chroma_to_string (wanted_info->chroma_site); + + if (GST_VIDEO_INFO_FORMAT (wanted_info) != + GST_VIDEO_INFO_FORMAT (current_info) + || g_strcmp0 (colorimetry, best_colorimetry) + || g_strcmp0 (chroma, best_chroma)) { + GstVideoInfo tmp_info = *current_info; + + tmp_info.finfo = wanted_info->finfo; + tmp_info.chroma_site = wanted_info->chroma_site; + tmp_info.colorimetry = wanted_info->colorimetry; + + GST_DEBUG_OBJECT (pad, "This pad will be converted from %d to %d", + GST_VIDEO_INFO_FORMAT (current_info), + GST_VIDEO_INFO_FORMAT (wanted_info)); + pad->priv->convert = + gst_video_converter_new (current_info, &tmp_info, NULL); + pad->need_conversion_update = TRUE; + if (!pad->priv->convert) { + g_free (colorimetry); + g_free (best_colorimetry); + GST_WARNING ("No path found for conversion"); + return FALSE; + } + } else { + GST_DEBUG_OBJECT (pad, "This pad will not need conversion"); + } + g_free (colorimetry); + g_free (best_colorimetry); + + return TRUE; +} + static void gst_videoaggregator_pad_finalize (GObject * o) { @@ -143,6 +200,79 @@ gst_videoaggregator_pad_finalize (GObject * o) G_OBJECT_CLASS (gst_videoaggregator_pad_parent_class)->finalize (o); } +static gboolean +gst_video_aggregator_pad_prepare_frame (GstVideoAggregatorPad * pad, + GstVideoAggregator * vagg) +{ + guint outsize; + GstVideoFrame *converted_frame; + GstBuffer *converted_buf = NULL; + GstVideoFrame *frame = g_slice_new0 (GstVideoFrame); + static GstAllocationParams params = { 0, 15, 0, 0, }; + + if (!gst_video_frame_map (frame, &pad->buffer_vinfo, pad->buffer, + GST_MAP_READ)) { + GST_WARNING_OBJECT (vagg, "Could not map input buffer"); + } + + if (pad->priv->convert) { + gint converted_size; + + converted_frame = g_slice_new0 (GstVideoFrame); + + /* We wait until here to set the conversion infos, in case vagg->info changed */ + if (pad->need_conversion_update) { + pad->conversion_info = vagg->info; + gst_video_info_set_format (&(pad->conversion_info), + GST_VIDEO_INFO_FORMAT (&vagg->info), pad->info.width, + pad->info.height); + pad->need_conversion_update = FALSE; + } + + converted_size = pad->conversion_info.size; + outsize = GST_VIDEO_INFO_SIZE (&vagg->info); + converted_size = converted_size > outsize ? converted_size : outsize; + converted_buf = gst_buffer_new_allocate (NULL, converted_size, ¶ms); + + if (!gst_video_frame_map (converted_frame, &(pad->conversion_info), + converted_buf, GST_MAP_READWRITE)) { + GST_WARNING_OBJECT (vagg, "Could not map converted frame"); + + g_slice_free (GstVideoFrame, converted_frame); + gst_video_frame_unmap (frame); + g_slice_free (GstVideoFrame, frame); + return FALSE; + } + + gst_video_converter_frame (pad->priv->convert, frame, converted_frame); + pad->converted_buffer = converted_buf; + gst_video_frame_unmap (frame); + g_slice_free (GstVideoFrame, frame); + } else { + converted_frame = frame; + } + + pad->aggregated_frame = converted_frame; + + return TRUE; +} + +static void +gst_video_aggregator_pad_clean_frame (GstVideoAggregatorPad * pad, + GstVideoAggregator * vagg) +{ + if (pad->aggregated_frame) { + gst_video_frame_unmap (pad->aggregated_frame); + g_slice_free (GstVideoFrame, pad->aggregated_frame); + pad->aggregated_frame = NULL; + } + + if (pad->converted_buffer) { + gst_buffer_unref (pad->converted_buffer); + pad->converted_buffer = NULL; + } +} + static void gst_videoaggregator_pad_class_init (GstVideoAggregatorPadClass * klass) { @@ -161,6 +291,10 @@ gst_videoaggregator_pad_class_init (GstVideoAggregatorPadClass * klass) g_type_class_add_private (klass, sizeof (GstVideoAggregatorPadPrivate)); aggpadclass->flush = GST_DEBUG_FUNCPTR (_flush_pad); + klass->set_info = GST_DEBUG_FUNCPTR (gst_video_aggregator_pad_set_info); + klass->prepare_frame = + GST_DEBUG_FUNCPTR (gst_video_aggregator_pad_prepare_frame); + klass->clean_frame = GST_DEBUG_FUNCPTR (gst_video_aggregator_pad_clean_frame); } static void @@ -268,8 +402,8 @@ G_DEFINE_ABSTRACT_TYPE_WITH_CODE (GstVideoAggregator, gst_videoaggregator, gst_videoaggregator_child_proxy_init)); static void -_find_best_video_format (GstVideoAggregator * vagg, GstCaps * downstream_caps, - GstVideoInfo * best_info, GstVideoFormat * best_format, +gst_videoaggreagator_find_best_format (GstVideoAggregator * vagg, + GstCaps * downstream_caps, GstVideoInfo * best_info, gboolean * at_least_one_alpha) { GList *tmp; @@ -326,11 +460,9 @@ _find_best_video_format (GstVideoAggregator * vagg, GstCaps * downstream_caps, /* If that pad is the first with alpha, set it as the new best format */ if (!need_alpha && (pad->info.finfo->flags & GST_VIDEO_FORMAT_FLAG_ALPHA)) { need_alpha = TRUE; - *best_format = GST_VIDEO_INFO_FORMAT (&pad->info); *best_info = pad->info; best_format_number = format_number; } else if (format_number > best_format_number) { - *best_format = GST_VIDEO_INFO_FORMAT (&pad->info); *best_info = pad->info; best_format_number = format_number; } @@ -349,17 +481,16 @@ static gboolean gst_videoaggregator_update_converters (GstVideoAggregator * vagg) { GList *tmp; - GstVideoAggregatorPad *pad; GstVideoFormat best_format; GstVideoInfo best_info; gboolean at_least_one_alpha = FALSE; GstCaps *downstream_caps; - gchar *best_colorimetry; - const gchar *best_chroma; - GstElementClass *klass = GST_ELEMENT_GET_CLASS (vagg); - GstVideoAggregatorClass *vagg_klass = (GstVideoAggregatorClass *) klass; GstAggregator *agg = GST_AGGREGATOR (vagg); + GstVideoAggregatorClass *vagg_class = GST_VIDEO_AGGREGATOR_GET_CLASS (vagg); + GstVideoAggregatorPadClass *vaggpad_class = g_type_class_peek + (GST_AGGREGATOR_GET_CLASS (vagg)->sinkpads_type); + best_format = GST_VIDEO_FORMAT_UNKNOWN; gst_video_info_init (&best_info); @@ -374,10 +505,13 @@ gst_videoaggregator_update_converters (GstVideoAggregator * vagg) } - if (vagg_klass->disable_frame_conversion == FALSE) - _find_best_video_format (vagg, downstream_caps, &best_info, &best_format, + if (vagg_class->find_best_format) { + vagg_class->find_best_format (vagg, downstream_caps, &best_info, &at_least_one_alpha); + best_format = GST_VIDEO_INFO_FORMAT (&best_info); + } + if (best_format == GST_VIDEO_FORMAT_UNKNOWN) { downstream_caps = gst_caps_fixate (downstream_caps); gst_video_info_from_caps (&best_info, downstream_caps); @@ -396,68 +530,25 @@ gst_videoaggregator_update_converters (GstVideoAggregator * vagg) vagg->info = best_info; - /* short circuit */ - if (vagg_klass->disable_frame_conversion) - return TRUE; - - best_colorimetry = gst_video_colorimetry_to_string (&(best_info.colorimetry)); - best_chroma = gst_video_chroma_to_string (best_info.chroma_site); - GST_DEBUG_OBJECT (vagg, - "The output format will now be : %d with colorimetry : %s and chroma : %s", - best_format, best_colorimetry, best_chroma); + "The output format will now be : %d with chroma : %s", + best_format, gst_video_chroma_to_string (best_info.chroma_site)); - /* Then browse the sinks once more, setting or unsetting conversion if needed */ - GST_OBJECT_LOCK (vagg); - for (tmp = GST_ELEMENT (vagg)->sinkpads; tmp; tmp = tmp->next) { - gchar *colorimetry; - const gchar *chroma; + if (vaggpad_class->set_info) { + GST_OBJECT_LOCK (vagg); + /* Then browse the sinks once more, setting or unsetting conversion if needed */ + for (tmp = GST_ELEMENT (vagg)->sinkpads; tmp; tmp = tmp->next) { + GstVideoAggregatorPad *pad = GST_VIDEO_AGGREGATOR_PAD (tmp->data); - pad = tmp->data; - - if (!pad->info.finfo) - continue; - - if (GST_VIDEO_INFO_FORMAT (&pad->info) == GST_VIDEO_FORMAT_UNKNOWN) - continue; - - if (pad->priv->convert) - gst_video_converter_free (pad->priv->convert); - - pad->priv->convert = NULL; - - colorimetry = gst_video_colorimetry_to_string (&(pad->info.colorimetry)); - chroma = gst_video_chroma_to_string (pad->info.chroma_site); - - if (best_format != GST_VIDEO_INFO_FORMAT (&pad->info) || - g_strcmp0 (colorimetry, best_colorimetry) || - g_strcmp0 (chroma, best_chroma)) { - GstVideoInfo tmp_info = pad->info; - tmp_info.finfo = best_info.finfo; - tmp_info.chroma_site = best_info.chroma_site; - tmp_info.colorimetry = best_info.colorimetry; - - GST_DEBUG_OBJECT (pad, "This pad will be converted from %d to %d", - GST_VIDEO_INFO_FORMAT (&pad->info), - GST_VIDEO_INFO_FORMAT (&best_info)); - pad->priv->convert = - gst_video_converter_new (&pad->info, &tmp_info, NULL); - pad->need_conversion_update = TRUE; - if (!pad->priv->convert) { - g_free (colorimetry); - g_free (best_colorimetry); - GST_WARNING ("No path found for conversion"); + if (!vaggpad_class->set_info (pad, vagg, &pad->info, &best_info)) { GST_OBJECT_UNLOCK (vagg); + return FALSE; } - } else { - GST_DEBUG_OBJECT (pad, "This pad will not need conversion"); } - g_free (colorimetry); + GST_OBJECT_UNLOCK (vagg); } - GST_OBJECT_UNLOCK (vagg); - g_free (best_colorimetry); return TRUE; } @@ -990,7 +1081,8 @@ static gboolean prepare_frames (GstVideoAggregator * vagg, GstVideoAggregatorPad * pad) { GstAggregatorPad *bpad = GST_AGGREGATOR_PAD (pad); - GstVideoAggregatorClass *vagg_klass = GST_VIDEO_AGGREGATOR_GET_CLASS (vagg); + GstVideoAggregatorPadClass *vaggpad_class = + GST_VIDEO_AGGREGATOR_PAD_GET_CLASS (pad); if (pad->buffer != NULL) { GstClockTime timestamp; @@ -1007,56 +1099,8 @@ prepare_frames (GstVideoAggregator * vagg, GstVideoAggregatorPad * pad) if (GST_CLOCK_TIME_IS_VALID (stream_time)) gst_object_sync_values (GST_OBJECT (pad), stream_time); - if (!vagg_klass->disable_frame_conversion) { - guint outsize; - GstVideoFrame *converted_frame; - GstBuffer *converted_buf = NULL; - GstVideoFrame *frame = g_slice_new0 (GstVideoFrame); - static GstAllocationParams params = { 0, 15, 0, 0, }; - - if (!gst_video_frame_map (frame, &pad->buffer_vinfo, pad->buffer, - GST_MAP_READ)) { - GST_WARNING_OBJECT (vagg, "Could not map input buffer"); - } - - if (pad->priv->convert) { - gint converted_size; - - converted_frame = g_slice_new0 (GstVideoFrame); - - /* We wait until here to set the conversion infos, in case vagg->info changed */ - if (pad->need_conversion_update) { - pad->conversion_info = vagg->info; - gst_video_info_set_format (&(pad->conversion_info), - GST_VIDEO_INFO_FORMAT (&vagg->info), pad->info.width, - pad->info.height); - pad->need_conversion_update = FALSE; - } - - converted_size = pad->conversion_info.size; - outsize = GST_VIDEO_INFO_SIZE (&vagg->info); - converted_size = converted_size > outsize ? converted_size : outsize; - converted_buf = gst_buffer_new_allocate (NULL, converted_size, ¶ms); - - if (!gst_video_frame_map (converted_frame, &(pad->conversion_info), - converted_buf, GST_MAP_READWRITE)) { - GST_WARNING_OBJECT (vagg, "Could not map converted frame"); - - g_slice_free (GstVideoFrame, converted_frame); - gst_video_frame_unmap (frame); - g_slice_free (GstVideoFrame, frame); - return FALSE; - } - - gst_video_converter_frame (pad->priv->convert, frame, converted_frame); - pad->converted_buffer = converted_buf; - gst_video_frame_unmap (frame); - g_slice_free (GstVideoFrame, frame); - } else { - converted_frame = frame; - } - - pad->aggregated_frame = converted_frame; + if (vaggpad_class->prepare_frame) { + return vaggpad_class->prepare_frame (pad, vagg); } } @@ -1066,16 +1110,10 @@ prepare_frames (GstVideoAggregator * vagg, GstVideoAggregatorPad * pad) static gboolean clean_pad (GstVideoAggregator * vagg, GstVideoAggregatorPad * pad) { - if (pad->aggregated_frame) { - gst_video_frame_unmap (pad->aggregated_frame); - g_slice_free (GstVideoFrame, pad->aggregated_frame); - pad->aggregated_frame = NULL; - } + GstVideoAggregatorPadClass *vaggpad_class = + GST_VIDEO_AGGREGATOR_PAD_GET_CLASS (pad); - if (pad->converted_buffer) { - gst_buffer_unref (pad->converted_buffer); - pad->converted_buffer = NULL; - } + vaggpad_class->clean_frame (pad, vagg); return TRUE; } @@ -1088,6 +1126,8 @@ gst_videoaggregator_do_aggregate (GstVideoAggregator * vagg, GstFlowReturn ret = GST_FLOW_OK; GstElementClass *klass = GST_ELEMENT_GET_CLASS (vagg); GstVideoAggregatorClass *vagg_klass = (GstVideoAggregatorClass *) klass; + GstVideoAggregatorPadClass *vaggpad_class = g_type_class_peek + (GST_AGGREGATOR_CLASS (klass)->sinkpads_type); g_assert (vagg_klass->aggregate_frames != NULL); g_assert (vagg_klass->get_output_buffer != NULL); @@ -1108,8 +1148,10 @@ gst_videoaggregator_do_aggregate (GstVideoAggregator * vagg, ret = vagg_klass->aggregate_frames (vagg, *outbuf); - gst_aggregator_iterate_sinkpads (GST_AGGREGATOR (vagg), - (GstAggregatorPadForeachFunc) clean_pad, NULL); + if (vaggpad_class->clean_frame) { + gst_aggregator_iterate_sinkpads (GST_AGGREGATOR (vagg), + (GstAggregatorPadForeachFunc) clean_pad, NULL); + } return ret; } @@ -1838,6 +1880,7 @@ gst_videoaggregator_class_init (GstVideoAggregatorClass * klass) agg_class->src_event = gst_videoaggregator_src_event; agg_class->src_query = gst_videoaggregator_src_query; + klass->find_best_format = gst_videoaggreagator_find_best_format; klass->get_output_buffer = gst_videoaggregator_get_output_buffer; /* Register the pad class */ diff --git a/gst-libs/gst/video/gstvideoaggregator.h b/gst-libs/gst/video/gstvideoaggregator.h index 2bdd6195f8..60733ed2c9 100644 --- a/gst-libs/gst/video/gstvideoaggregator.h +++ b/gst-libs/gst/video/gstvideoaggregator.h @@ -30,8 +30,6 @@ #include #include -#include "gstvideoaggregatorpad.h" - G_BEGIN_DECLS #define GST_TYPE_VIDEO_AGGREGATOR (gst_videoaggregator_get_type()) @@ -50,6 +48,8 @@ typedef struct _GstVideoAggregator GstVideoAggregator; typedef struct _GstVideoAggregatorClass GstVideoAggregatorClass; typedef struct _GstVideoAggregatorPrivate GstVideoAggregatorPrivate; +#include "gstvideoaggregatorpad.h" + /** * GstVideoAggregator: * @info: The #GstVideoInfo representing the currently set @@ -70,9 +70,6 @@ struct _GstVideoAggregator /** * GstVideoAggregatorClass: - * @disable_frame_conversion: Optional. - * Allows subclasses to disable the frame colorspace - * conversion feature * @update_caps: Optional. * Lets subclasses update the #GstCaps representing * the src pad caps before usage. Return %NULL to indicate failure. @@ -87,6 +84,8 @@ struct _GstVideoAggregator * the #aggregate_frames vmethod. * @negotiated_caps: Optional. * Notifies subclasses what caps format has been negotiated + * @find_best_format: Optional. + * Lets subclasses decide of the best common format to use. **/ struct _GstVideoAggregatorClass { @@ -94,8 +93,6 @@ struct _GstVideoAggregatorClass GstAggregatorClass parent_class; /*< public >*/ - gboolean disable_frame_conversion; - GstCaps * (*update_caps) (GstVideoAggregator * videoaggregator, GstCaps * caps); GstFlowReturn (*aggregate_frames) (GstVideoAggregator * videoaggregator, @@ -104,6 +101,10 @@ struct _GstVideoAggregatorClass GstBuffer ** outbuffer); gboolean (*negotiated_caps) (GstVideoAggregator * videoaggregator, GstCaps * caps); + void (*find_best_format) (GstVideoAggregator * vagg, + GstCaps * downstream_caps, + GstVideoInfo * best_info, + gboolean * at_least_one_alpha); /* < private > */ gpointer _gst_reserved[GST_PADDING]; }; From ec45cbda222e7c7c715f72f731576c4c1ed4e2cf Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Thu, 27 Nov 2014 18:46:03 +0100 Subject: [PATCH 082/381] videoconvert: Hide all conversion related fields And do not delay the setting of the conversion_info https://bugzilla.gnome.org/show_bug.cgi?id=740768 --- gst-libs/gst/video/gstvideoaggregator.c | 31 +++++++++++-------------- 1 file changed, 14 insertions(+), 17 deletions(-) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index 9e9bb3a345..c640e67f0b 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -65,6 +65,10 @@ struct _GstVideoAggregatorPadPrivate { /* Converter, if NULL no conversion is done */ GstVideoConverter *convert; + + /* caps used for conversion if needed */ + GstVideoInfo conversion_info; + GstBuffer *converted_buffer; }; G_DEFINE_TYPE (GstVideoAggregatorPad, gst_videoaggregator_pad, @@ -172,7 +176,9 @@ gst_video_aggregator_pad_set_info (GstVideoAggregatorPad * pad, GST_VIDEO_INFO_FORMAT (wanted_info)); pad->priv->convert = gst_video_converter_new (current_info, &tmp_info, NULL); - pad->need_conversion_update = TRUE; + pad->priv->conversion_info = vagg->info; + gst_video_info_set_format (&(pad->priv->conversion_info), + GST_VIDEO_INFO_FORMAT (&vagg->info), pad->info.width, pad->info.height); if (!pad->priv->convert) { g_free (colorimetry); g_free (best_colorimetry); @@ -221,20 +227,12 @@ gst_video_aggregator_pad_prepare_frame (GstVideoAggregatorPad * pad, converted_frame = g_slice_new0 (GstVideoFrame); /* We wait until here to set the conversion infos, in case vagg->info changed */ - if (pad->need_conversion_update) { - pad->conversion_info = vagg->info; - gst_video_info_set_format (&(pad->conversion_info), - GST_VIDEO_INFO_FORMAT (&vagg->info), pad->info.width, - pad->info.height); - pad->need_conversion_update = FALSE; - } - - converted_size = pad->conversion_info.size; + converted_size = pad->priv->conversion_info.size; outsize = GST_VIDEO_INFO_SIZE (&vagg->info); converted_size = converted_size > outsize ? converted_size : outsize; converted_buf = gst_buffer_new_allocate (NULL, converted_size, ¶ms); - if (!gst_video_frame_map (converted_frame, &(pad->conversion_info), + if (!gst_video_frame_map (converted_frame, &(pad->priv->conversion_info), converted_buf, GST_MAP_READWRITE)) { GST_WARNING_OBJECT (vagg, "Could not map converted frame"); @@ -245,7 +243,7 @@ gst_video_aggregator_pad_prepare_frame (GstVideoAggregatorPad * pad, } gst_video_converter_frame (pad->priv->convert, frame, converted_frame); - pad->converted_buffer = converted_buf; + pad->priv->converted_buffer = converted_buf; gst_video_frame_unmap (frame); g_slice_free (GstVideoFrame, frame); } else { @@ -267,9 +265,9 @@ gst_video_aggregator_pad_clean_frame (GstVideoAggregatorPad * pad, pad->aggregated_frame = NULL; } - if (pad->converted_buffer) { - gst_buffer_unref (pad->converted_buffer); - pad->converted_buffer = NULL; + if (pad->priv->converted_buffer) { + gst_buffer_unref (pad->priv->converted_buffer); + pad->priv->converted_buffer = NULL; } } @@ -305,9 +303,8 @@ gst_videoaggregator_pad_init (GstVideoAggregatorPad * vaggpad) GstVideoAggregatorPadPrivate); vaggpad->zorder = DEFAULT_PAD_ZORDER; - vaggpad->need_conversion_update = FALSE; vaggpad->aggregated_frame = NULL; - vaggpad->converted_buffer = NULL; + vaggpad->priv->converted_buffer = NULL; vaggpad->priv->convert = NULL; } From 00880b5830060b6733a9332a4163bab407d2dfce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Thu, 27 Nov 2014 19:52:20 +0100 Subject: [PATCH 083/381] videoaggregator: Do source pad negotiation only from the aggregated function Otherwise we might negotiate from the sinkpad streaming threads at the same time as on the srcpad streaming thread, and then all kinds of crazy bugs happen that don't make any sense at all. --- gst-libs/gst/video/gstvideoaggregator.c | 36 +++++++++++-------------- 1 file changed, 16 insertions(+), 20 deletions(-) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index c640e67f0b..8af97ce9ba 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -770,11 +770,8 @@ gst_videoaggregator_pad_sink_setcaps (GstPad * pad, GstObject * parent, } vaggpad->info = info; - - ret = gst_videoaggregator_update_converters (vagg); - - if (ret) - ret = gst_videoaggregator_update_src_caps (vagg); + gst_pad_mark_reconfigure (vagg->aggregator.srcpad); + ret = TRUE; GST_VIDEO_AGGREGATOR_UNLOCK (vagg); @@ -1206,16 +1203,19 @@ gst_videoaggregator_aggregate (GstAggregator * agg) gint res; gint64 jitter; - /* If we're not negotiated_caps yet... */ - if (GST_VIDEO_INFO_FORMAT (&vagg->info) == GST_VIDEO_FORMAT_UNKNOWN) { - GST_INFO_OBJECT (agg, "Not negotiated yet!"); - return GST_FLOW_NOT_NEGOTIATED; - } - GST_VIDEO_AGGREGATOR_LOCK (vagg); - if (gst_pad_check_reconfigure (agg->srcpad)) - gst_videoaggregator_update_src_caps (vagg); + if (GST_VIDEO_INFO_FORMAT (&vagg->info) == GST_VIDEO_FORMAT_UNKNOWN + || gst_pad_check_reconfigure (agg->srcpad)) { + ret = gst_videoaggregator_update_converters (vagg); + if (ret) + ret = gst_videoaggregator_update_src_caps (vagg); + + if (!ret) { + GST_VIDEO_AGGREGATOR_UNLOCK (vagg); + return GST_FLOW_NOT_NEGOTIATED; + } + } if (agg->segment.position == -1) output_start_time = agg->segment.start; @@ -1648,7 +1648,7 @@ gst_videoaggregator_release_pad (GstElement * element, GstPad * pad) { GstVideoAggregator *vagg = NULL; GstVideoAggregatorPad *vaggpad; - gboolean update_caps, last_pad; + gboolean last_pad; vagg = GST_VIDEO_AGGREGATOR (element); vaggpad = GST_VIDEO_AGGREGATOR_PAD (pad); @@ -1659,13 +1659,10 @@ gst_videoaggregator_release_pad (GstElement * element, GstPad * pad) last_pad = (GST_ELEMENT (vagg)->numsinkpads - 1 == 0); GST_OBJECT_UNLOCK (vagg); - if (!last_pad) - gst_videoaggregator_update_converters (vagg); - else + if (last_pad) gst_videoaggregator_reset (vagg); gst_buffer_replace (&vaggpad->buffer, NULL); - update_caps = GST_VIDEO_INFO_FORMAT (&vagg->info) != GST_VIDEO_FORMAT_UNKNOWN; gst_child_proxy_child_removed (GST_CHILD_PROXY (vagg), G_OBJECT (vaggpad), GST_OBJECT_NAME (vaggpad)); @@ -1673,8 +1670,7 @@ gst_videoaggregator_release_pad (GstElement * element, GstPad * pad) GST_ELEMENT_CLASS (gst_videoaggregator_parent_class)->release_pad (GST_ELEMENT (vagg), pad); - if (update_caps) - gst_videoaggregator_update_src_caps (vagg); + gst_pad_mark_reconfigure (vagg->aggregator.srcpad); GST_VIDEO_AGGREGATOR_UNLOCK (vagg); return; From 4233ba4ca4f9679426e28f4ec26dc4bc5c515895 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Thu, 27 Nov 2014 20:25:29 +0100 Subject: [PATCH 084/381] videoaggregator: Directly use the converters video-info instead of recalculating it --- gst-libs/gst/video/gstvideoaggregator.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index 8af97ce9ba..47e137c625 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -176,9 +176,7 @@ gst_video_aggregator_pad_set_info (GstVideoAggregatorPad * pad, GST_VIDEO_INFO_FORMAT (wanted_info)); pad->priv->convert = gst_video_converter_new (current_info, &tmp_info, NULL); - pad->priv->conversion_info = vagg->info; - gst_video_info_set_format (&(pad->priv->conversion_info), - GST_VIDEO_INFO_FORMAT (&vagg->info), pad->info.width, pad->info.height); + pad->priv->conversion_info = tmp_info; if (!pad->priv->convert) { g_free (colorimetry); g_free (best_colorimetry); From fdc87a52f28c8c54702b2ef6deee7f730de192cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Thu, 27 Nov 2014 20:34:25 +0100 Subject: [PATCH 085/381] videoaggregator: Copy over more fields from the relevant video-info gst_video_info_set_format() will reset the complete video-info, but we want to keep values like the PAR, colorimetry and chroma site. Otherwise we risk setting different values on the srcpad caps than what is actually inside the buffers. --- gst-libs/gst/video/gstvideoaggregator.c | 30 ++++++++++++++++++++----- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index 47e137c625..e294dffb06 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -165,11 +165,23 @@ gst_video_aggregator_pad_set_info (GstVideoAggregatorPad * pad, GST_VIDEO_INFO_FORMAT (current_info) || g_strcmp0 (colorimetry, best_colorimetry) || g_strcmp0 (chroma, best_chroma)) { - GstVideoInfo tmp_info = *current_info; + GstVideoInfo tmp_info; - tmp_info.finfo = wanted_info->finfo; + /* Initialize with the wanted video format and our original width and + * height as we don't want to rescale. Then copy over the wanted + * colorimetry, and chroma-site and our current pixel-aspect-ratio + * and other relevant fields. + */ + gst_video_info_set_format (&tmp_info, GST_VIDEO_INFO_FORMAT (wanted_info), + current_info->width, current_info->height); tmp_info.chroma_site = wanted_info->chroma_site; tmp_info.colorimetry = wanted_info->colorimetry; + tmp_info.par_n = current_info->par_n; + tmp_info.par_d = current_info->par_d; + tmp_info.fps_n = current_info->fps_n; + tmp_info.fps_d = current_info->fps_d; + tmp_info.flags = current_info->flags; + tmp_info.interlace_mode = current_info->interlace_mode; GST_DEBUG_OBJECT (pad, "This pad will be converted from %d to %d", GST_VIDEO_INFO_FORMAT (current_info), @@ -656,13 +668,21 @@ gst_videoaggregator_update_src_caps (GstVideoAggregator * vagg) vagg->priv->nframes = 0; } } - gst_video_info_init (&info); + + /* Initialize the video info with our target format and + * the best width and height and framerate. Then copy over + * all other fields as we negotiated them before + */ gst_video_info_set_format (&info, GST_VIDEO_INFO_FORMAT (&vagg->info), best_width, best_height); info.fps_n = best_fps_n; info.fps_d = best_fps_d; - info.par_n = GST_VIDEO_INFO_PAR_N (&vagg->info); - info.par_d = GST_VIDEO_INFO_PAR_D (&vagg->info); + info.chroma_site = vagg->info.chroma_site; + info.par_n = vagg->info.par_n; + info.par_d = vagg->info.par_d; + info.colorimetry = vagg->info.colorimetry; + info.flags = vagg->info.flags; + info.interlace_mode = vagg->info.interlace_mode; info_caps = gst_video_info_to_caps (&info); From 3e9b001013b4335b48316de676a43a08f0ecdaed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Wed, 26 Nov 2014 15:02:14 +0100 Subject: [PATCH 086/381] compositor: Implement rescaling of the input via pad properties compositor has now the same interface as glvideomixer. --- gst/compositor/compositor.c | 282 ++++++++++++++++++++++++++++++++- gst/compositor/compositorpad.h | 5 + 2 files changed, 285 insertions(+), 2 deletions(-) diff --git a/gst/compositor/compositor.c b/gst/compositor/compositor.c index b7848b47a7..1befdcc39e 100644 --- a/gst/compositor/compositor.c +++ b/gst/compositor/compositor.c @@ -114,12 +114,16 @@ static GstStaticPadTemplate sink_factory = GST_STATIC_PAD_TEMPLATE ("sink_%u", #define DEFAULT_PAD_XPOS 0 #define DEFAULT_PAD_YPOS 0 +#define DEFAULT_PAD_WIDTH 0 +#define DEFAULT_PAD_HEIGHT 0 #define DEFAULT_PAD_ALPHA 1.0 enum { PROP_PAD_0, PROP_PAD_XPOS, PROP_PAD_YPOS, + PROP_PAD_WIDTH, + PROP_PAD_HEIGHT, PROP_PAD_ALPHA }; @@ -139,6 +143,12 @@ gst_compositor_pad_get_property (GObject * object, guint prop_id, case PROP_PAD_YPOS: g_value_set_int (value, pad->ypos); break; + case PROP_PAD_WIDTH: + g_value_set_int (value, pad->width); + break; + case PROP_PAD_HEIGHT: + g_value_set_int (value, pad->height); + break; case PROP_PAD_ALPHA: g_value_set_double (value, pad->alpha); break; @@ -161,6 +171,12 @@ gst_compositor_pad_set_property (GObject * object, guint prop_id, case PROP_PAD_YPOS: pad->ypos = g_value_get_int (value); break; + case PROP_PAD_WIDTH: + pad->width = g_value_get_int (value); + break; + case PROP_PAD_HEIGHT: + pad->height = g_value_get_int (value); + break; case PROP_PAD_ALPHA: pad->alpha = g_value_get_double (value); break; @@ -170,13 +186,254 @@ gst_compositor_pad_set_property (GObject * object, guint prop_id, } } +static gboolean +gst_compositor_pad_set_info (GstVideoAggregatorPad * pad, + GstVideoAggregator * vagg G_GNUC_UNUSED, + GstVideoInfo * current_info, GstVideoInfo * wanted_info) +{ + GstCompositorPad *cpad = GST_COMPOSITOR_PAD (pad); + gchar *colorimetry, *best_colorimetry; + const gchar *chroma, *best_chroma; + gint width, height; + + if (!current_info->finfo) + return TRUE; + + if (GST_VIDEO_INFO_FORMAT (current_info) == GST_VIDEO_FORMAT_UNKNOWN) + return TRUE; + + if (cpad->convert) + gst_video_converter_free (cpad->convert); + + cpad->convert = NULL; + + colorimetry = gst_video_colorimetry_to_string (&(current_info->colorimetry)); + chroma = gst_video_chroma_to_string (current_info->chroma_site); + + best_colorimetry = + gst_video_colorimetry_to_string (&(wanted_info->colorimetry)); + best_chroma = gst_video_chroma_to_string (wanted_info->chroma_site); + + if (cpad->width > 0) + width = cpad->width; + else + width = current_info->width; + + if (cpad->height > 0) + height = cpad->height; + else + height = current_info->height; + + if (GST_VIDEO_INFO_FORMAT (wanted_info) != + GST_VIDEO_INFO_FORMAT (current_info) + || g_strcmp0 (colorimetry, best_colorimetry) + || g_strcmp0 (chroma, best_chroma) + || width != current_info->width || height != current_info->height) { + GstVideoInfo tmp_info; + + /* Initialize with the wanted video format and our original width and + * height as we don't want to rescale. Then copy over the wanted + * colorimetry, and chroma-site and our current pixel-aspect-ratio + * and other relevant fields. + */ + gst_video_info_set_format (&tmp_info, GST_VIDEO_INFO_FORMAT (wanted_info), + width, height); + tmp_info.chroma_site = wanted_info->chroma_site; + tmp_info.colorimetry = wanted_info->colorimetry; + tmp_info.par_n = current_info->par_n; + tmp_info.par_d = current_info->par_d; + tmp_info.fps_n = current_info->fps_n; + tmp_info.fps_d = current_info->fps_d; + tmp_info.flags = current_info->flags; + tmp_info.interlace_mode = current_info->interlace_mode; + + GST_DEBUG_OBJECT (pad, "This pad will be converted from %d to %d", + GST_VIDEO_INFO_FORMAT (current_info), + GST_VIDEO_INFO_FORMAT (&tmp_info)); + cpad->convert = gst_video_converter_new (current_info, &tmp_info, NULL); + cpad->conversion_info = tmp_info; + if (!cpad->convert) { + g_free (colorimetry); + g_free (best_colorimetry); + GST_WARNING_OBJECT (pad, "No path found for conversion"); + return FALSE; + } + } else { + cpad->conversion_info = *current_info; + GST_DEBUG_OBJECT (pad, "This pad will not need conversion"); + } + g_free (colorimetry); + g_free (best_colorimetry); + + return TRUE; +} + +static gboolean +gst_compositor_pad_prepare_frame (GstVideoAggregatorPad * pad, + GstVideoAggregator * vagg) +{ + GstCompositorPad *cpad = GST_COMPOSITOR_PAD (pad); + guint outsize; + GstVideoFrame *converted_frame; + GstBuffer *converted_buf = NULL; + GstVideoFrame *frame = g_slice_new0 (GstVideoFrame); + static GstAllocationParams params = { 0, 15, 0, 0, }; + gint width, height; + + if (!gst_video_frame_map (frame, &pad->buffer_vinfo, pad->buffer, + GST_MAP_READ)) { + GST_WARNING_OBJECT (vagg, "Could not map input buffer"); + } + + if (cpad->width > 0) + width = cpad->width; + else + width = GST_VIDEO_FRAME_WIDTH (frame); + + if (cpad->height > 0) + height = cpad->height; + else + height = GST_VIDEO_FRAME_HEIGHT (frame); + + /* The only thing that can change here is the width + * and height, otherwise set_info would've been called */ + if (cpad->conversion_info.width != width || + cpad->conversion_info.height != height) { + gchar *colorimetry, *wanted_colorimetry; + const gchar *chroma, *wanted_chroma; + + /* We might end up with no converter afterwards if + * the only reason for conversion was a different + * width or height + */ + if (cpad->convert) + gst_video_converter_free (cpad->convert); + cpad->convert = NULL; + + colorimetry = gst_video_colorimetry_to_string (&frame->info.colorimetry); + chroma = gst_video_chroma_to_string (frame->info.chroma_site); + + wanted_colorimetry = + gst_video_colorimetry_to_string (&cpad->conversion_info.colorimetry); + wanted_chroma = + gst_video_chroma_to_string (cpad->conversion_info.chroma_site); + + if (GST_VIDEO_INFO_FORMAT (&frame->info) != + GST_VIDEO_INFO_FORMAT (&cpad->conversion_info) + || g_strcmp0 (colorimetry, wanted_colorimetry) + || g_strcmp0 (chroma, wanted_chroma) + || width != GST_VIDEO_FRAME_WIDTH (frame) + || height != GST_VIDEO_FRAME_HEIGHT (frame)) { + GstVideoInfo tmp_info; + + gst_video_info_set_format (&tmp_info, cpad->conversion_info.finfo->format, + width, height); + tmp_info.chroma_site = cpad->conversion_info.chroma_site; + tmp_info.colorimetry = cpad->conversion_info.colorimetry; + tmp_info.par_n = cpad->conversion_info.par_n; + tmp_info.par_d = cpad->conversion_info.par_d; + tmp_info.fps_n = cpad->conversion_info.fps_n; + tmp_info.fps_d = cpad->conversion_info.fps_d; + tmp_info.flags = cpad->conversion_info.flags; + tmp_info.interlace_mode = cpad->conversion_info.interlace_mode; + + GST_DEBUG_OBJECT (pad, "This pad will be converted from %d to %d", + GST_VIDEO_INFO_FORMAT (&frame->info), + GST_VIDEO_INFO_FORMAT (&tmp_info)); + cpad->convert = gst_video_converter_new (&frame->info, &tmp_info, NULL); + cpad->conversion_info = tmp_info; + + if (!cpad->convert) { + GST_WARNING_OBJECT (pad, "No path found for conversion"); + g_free (colorimetry); + g_free (wanted_colorimetry); + gst_video_frame_unmap (frame); + g_slice_free (GstVideoFrame, frame); + return FALSE; + } + } else { + cpad->conversion_info.width = width; + cpad->conversion_info.height = height; + } + + g_free (colorimetry); + g_free (wanted_colorimetry); + } + + if (cpad->convert) { + gint converted_size; + + converted_frame = g_slice_new0 (GstVideoFrame); + + /* We wait until here to set the conversion infos, in case vagg->info changed */ + converted_size = cpad->conversion_info.size; + outsize = GST_VIDEO_INFO_SIZE (&vagg->info); + converted_size = converted_size > outsize ? converted_size : outsize; + converted_buf = gst_buffer_new_allocate (NULL, converted_size, ¶ms); + + if (!gst_video_frame_map (converted_frame, &(cpad->conversion_info), + converted_buf, GST_MAP_READWRITE)) { + GST_WARNING_OBJECT (vagg, "Could not map converted frame"); + + g_slice_free (GstVideoFrame, converted_frame); + gst_video_frame_unmap (frame); + g_slice_free (GstVideoFrame, frame); + return FALSE; + } + + gst_video_converter_frame (cpad->convert, frame, converted_frame); + cpad->converted_buffer = converted_buf; + gst_video_frame_unmap (frame); + g_slice_free (GstVideoFrame, frame); + } else { + converted_frame = frame; + } + + pad->aggregated_frame = converted_frame; + + return TRUE; +} + +static void +gst_compositor_pad_clean_frame (GstVideoAggregatorPad * pad, + GstVideoAggregator * vagg) +{ + GstCompositorPad *cpad = GST_COMPOSITOR_PAD (pad); + + if (pad->aggregated_frame) { + gst_video_frame_unmap (pad->aggregated_frame); + g_slice_free (GstVideoFrame, pad->aggregated_frame); + pad->aggregated_frame = NULL; + } + + if (cpad->converted_buffer) { + gst_buffer_unref (cpad->converted_buffer); + cpad->converted_buffer = NULL; + } +} + +static void +gst_compositor_pad_finalize (GObject * object) +{ + GstCompositorPad *pad = GST_COMPOSITOR_PAD (object); + + if (pad->convert) + gst_video_converter_free (pad->convert); + pad->convert = NULL; + + G_OBJECT_CLASS (gst_compositor_pad_parent_class)->finalize (object); +} + static void gst_compositor_pad_class_init (GstCompositorPadClass * klass) { GObjectClass *gobject_class = (GObjectClass *) klass; + GstVideoAggregatorPadClass *vaggpadclass = + (GstVideoAggregatorPadClass *) klass; gobject_class->set_property = gst_compositor_pad_set_property; gobject_class->get_property = gst_compositor_pad_get_property; + gobject_class->finalize = gst_compositor_pad_finalize; g_object_class_install_property (gobject_class, PROP_PAD_XPOS, g_param_spec_int ("xpos", "X Position", "X Position of the picture", @@ -186,10 +443,24 @@ gst_compositor_pad_class_init (GstCompositorPadClass * klass) g_param_spec_int ("ypos", "Y Position", "Y Position of the picture", G_MININT, G_MAXINT, DEFAULT_PAD_YPOS, G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); + g_object_class_install_property (gobject_class, PROP_PAD_WIDTH, + g_param_spec_int ("width", "Width", "Width of the picture", + G_MININT, G_MAXINT, DEFAULT_PAD_WIDTH, + G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); + g_object_class_install_property (gobject_class, PROP_PAD_HEIGHT, + g_param_spec_int ("height", "Height", "Height of the picture", + G_MININT, G_MAXINT, DEFAULT_PAD_HEIGHT, + G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); g_object_class_install_property (gobject_class, PROP_PAD_ALPHA, g_param_spec_double ("alpha", "Alpha", "Alpha of the picture", 0.0, 1.0, DEFAULT_PAD_ALPHA, G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); + + vaggpadclass->set_info = GST_DEBUG_FUNCPTR (gst_compositor_pad_set_info); + vaggpadclass->prepare_frame = + GST_DEBUG_FUNCPTR (gst_compositor_pad_prepare_frame); + vaggpadclass->clean_frame = + GST_DEBUG_FUNCPTR (gst_compositor_pad_clean_frame); } static void @@ -449,8 +720,15 @@ _update_caps (GstVideoAggregator * vagg, GstCaps * caps) gint this_width, this_height; gint width, height; - width = GST_VIDEO_INFO_WIDTH (&vaggpad->info); - height = GST_VIDEO_INFO_HEIGHT (&vaggpad->info); + if (compositor_pad->width > 0) + width = compositor_pad->width; + else + width = GST_VIDEO_INFO_WIDTH (&vaggpad->info); + + if (compositor_pad->height > 0) + height = compositor_pad->height; + else + height = GST_VIDEO_INFO_HEIGHT (&vaggpad->info); if (width == 0 || height == 0) continue; diff --git a/gst/compositor/compositorpad.h b/gst/compositor/compositorpad.h index a1c83f8d53..cb6f7c7356 100644 --- a/gst/compositor/compositorpad.h +++ b/gst/compositor/compositorpad.h @@ -50,7 +50,12 @@ struct _GstCompositorPad /* properties */ gint xpos, ypos; + gint width, height; gdouble alpha; + + GstVideoConverter *convert; + GstVideoInfo conversion_info; + GstBuffer *converted_buffer; }; struct _GstCompositorPadClass From 0b73de023ce300400504a878371e8ace11df13c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Thu, 27 Nov 2014 21:22:44 +0100 Subject: [PATCH 087/381] videoaggregator: Minor cleanup --- gst-libs/gst/video/gstvideoaggregator.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index e294dffb06..91463b680c 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -185,14 +185,14 @@ gst_video_aggregator_pad_set_info (GstVideoAggregatorPad * pad, GST_DEBUG_OBJECT (pad, "This pad will be converted from %d to %d", GST_VIDEO_INFO_FORMAT (current_info), - GST_VIDEO_INFO_FORMAT (wanted_info)); + GST_VIDEO_INFO_FORMAT (&tmp_info)); pad->priv->convert = gst_video_converter_new (current_info, &tmp_info, NULL); pad->priv->conversion_info = tmp_info; if (!pad->priv->convert) { g_free (colorimetry); g_free (best_colorimetry); - GST_WARNING ("No path found for conversion"); + GST_WARNING_OBJECT (pad, "No path found for conversion"); return FALSE; } } else { From ad77f0392f1b70728780e3edc3e86b9efbba8a16 Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Fri, 14 Nov 2014 00:20:10 +1100 Subject: [PATCH 088/381] glvideomixer: add support for gl3 --- ext/gl/gstglvideomixer.c | 137 ++++++++++++++++++++++++++++++--------- ext/gl/gstglvideomixer.h | 3 + 2 files changed, 110 insertions(+), 30 deletions(-) diff --git a/ext/gl/gstglvideomixer.c b/ext/gl/gstglvideomixer.c index 39e17ae21d..d73bf97150 100644 --- a/ext/gl/gstglvideomixer.c +++ b/ext/gl/gstglvideomixer.c @@ -159,6 +159,9 @@ struct _GstGLVideoMixerPad gint xpos, ypos; gint width, height; gdouble alpha; + + gboolean geometry_change; + GLuint vertex_buffer; }; struct _GstGLVideoMixerPadClass @@ -264,15 +267,19 @@ gst_gl_video_mixer_pad_set_property (GObject * object, guint prop_id, switch (prop_id) { case PROP_PAD_XPOS: pad->xpos = g_value_get_int (value); + pad->geometry_change = TRUE; break; case PROP_PAD_YPOS: pad->ypos = g_value_get_int (value); + pad->geometry_change = TRUE; break; case PROP_PAD_WIDTH: pad->width = g_value_get_int (value); + pad->geometry_change = TRUE; break; case PROP_PAD_HEIGHT: pad->height = g_value_get_int (value); + pad->geometry_change = TRUE; break; case PROP_PAD_ALPHA: pad->alpha = g_value_get_double (value); @@ -434,6 +441,34 @@ _update_caps (GstVideoAggregator * vagg, GstCaps * caps) return ret; } +static gboolean +_reset_pad_gl (GstAggregator * agg, GstPad * aggpad, gpointer udata) +{ + const GstGLFuncs *gl = GST_GL_MIXER (agg)->context->gl_vtable; + GstGLVideoMixerPad *pad = GST_GL_VIDEO_MIXER_PAD (aggpad); + + if (pad->vertex_buffer) { + gl->DeleteBuffers (1, &pad->vertex_buffer); + pad->vertex_buffer = 0; + } + + return FALSE; +} + +static void +_reset_gl (GstGLContext * context, GstGLVideoMixer * video_mixer) +{ + const GstGLFuncs *gl = GST_GL_MIXER (video_mixer)->context->gl_vtable; + + if (video_mixer->vao) { + gl->DeleteVertexArrays (1, &video_mixer->vao); + video_mixer->vao = 0; + } + + gst_aggregator_iterate_sinkpads (GST_AGGREGATOR (video_mixer), _reset_pad_gl, + NULL); +} + static void gst_gl_video_mixer_reset (GstGLMixer * mixer) { @@ -448,6 +483,10 @@ gst_gl_video_mixer_reset (GstGLMixer * mixer) if (video_mixer->checker) gst_gl_context_del_shader (mixer->context, video_mixer->checker); video_mixer->checker = NULL; + + if (mixer->context) + gst_gl_context_thread_add (mixer->context, + (GstGLContextThreadFunc) _reset_gl, mixer); } static gboolean @@ -509,13 +548,25 @@ _draw_checker_background (GstGLVideoMixer * video_mixer) attr_position_loc = gst_gl_shader_get_attribute_location (video_mixer->checker, "a_position"); + if (!video_mixer->checker_vbo) { + gl->GenBuffers (1, &video_mixer->checker_vbo); + gl->BindBuffer (GL_ARRAY_BUFFER, video_mixer->checker_vbo); + gl->BufferData (GL_ARRAY_BUFFER, 4 * 3 * sizeof (GLfloat), v_vertices, + GL_STATIC_DRAW); + } else { + gl->BindBuffer (GL_ARRAY_BUFFER, video_mixer->checker_vbo); + } + gl->VertexAttribPointer (attr_position_loc, 3, GL_FLOAT, - GL_FALSE, 3 * sizeof (GLfloat), &v_vertices[0]); + GL_FALSE, 3 * sizeof (GLfloat), (void *) 0); gl->EnableVertexAttribArray (attr_position_loc); gl->DrawElements (GL_TRIANGLES, 6, GL_UNSIGNED_SHORT, indices); + gl->DisableVertexAttribArray (attr_position_loc); + gl->BindBuffer (GL_ARRAY_BUFFER, 0); + return TRUE; } @@ -572,11 +623,18 @@ gst_gl_video_mixer_callback (gpointer stuff) gst_gl_context_clear_shader (mixer->context); gl->BindTexture (GL_TEXTURE_2D, 0); - gl->Disable (GL_TEXTURE_2D); + if (gst_gl_context_get_gl_api (mixer->context) & GST_GL_API_OPENGL) + gl->Disable (GL_TEXTURE_2D); gl->Disable (GL_DEPTH_TEST); gl->Disable (GL_CULL_FACE); + if (gl->GenVertexArrays) { + if (!video_mixer->vao) + gl->GenVertexArrays (1, &video_mixer->vao); + gl->BindVertexArray (video_mixer->vao); + } + if (!_draw_background (video_mixer)) return; @@ -592,19 +650,17 @@ gst_gl_video_mixer_callback (gpointer stuff) while (count < video_mixer->input_frames->len) { GstGLMixerFrameData *frame; GstGLVideoMixerPad *pad; + guint in_tex; + guint in_width, in_height; + /* *INDENT-OFF* */ gfloat v_vertices[] = { - /* front face */ -1.0,-1.0,-1.0f, 0.0f, 0.0f, 1.0,-1.0,-1.0f, 1.0f, 0.0f, 1.0, 1.0,-1.0f, 1.0f, 1.0f, -1.0, 1.0,-1.0f, 0.0f, 1.0f, }; /* *INDENT-ON* */ - guint in_tex; - guint in_width, in_height; - guint pad_width, pad_height; - gfloat w, h; frame = g_ptr_array_index (video_mixer->input_frames, count); if (!frame) { @@ -625,34 +681,41 @@ gst_gl_video_mixer_callback (gpointer stuff) } in_tex = frame->texture; - pad_width = pad->width <= 0 ? in_width : pad->width; - pad_height = pad->height <= 0 ? in_height : pad->height; - w = ((gfloat) pad_width / (gfloat) out_width); - h = ((gfloat) pad_height / (gfloat) out_height); + if (pad->geometry_change || !pad->vertex_buffer) { + guint pad_width, pad_height; + gfloat w, h; - /* top-left */ - v_vertices[0] = v_vertices[15] = - 2.0f * (gfloat) pad->xpos / (gfloat) out_width - 1.0f; - /* bottom-left */ - v_vertices[1] = v_vertices[6] = - 2.0f * (gfloat) pad->ypos / (gfloat) out_height - 1.0f; - /* top-right */ - v_vertices[5] = v_vertices[10] = v_vertices[0] + 2.0f * w; - /* bottom-right */ - v_vertices[11] = v_vertices[16] = v_vertices[1] + 2.0f * h; - GST_TRACE ("processing texture:%u dimensions:%ux%u, at %f,%f %fx%f with " - "alpha:%f", in_tex, in_width, in_height, v_vertices[0], v_vertices[1], - v_vertices[5], v_vertices[11], pad->alpha); + pad_width = pad->width <= 0 ? in_width : pad->width; + pad_height = pad->height <= 0 ? in_height : pad->height; - gl->VertexAttribPointer (attr_position_loc, 3, GL_FLOAT, - GL_FALSE, 5 * sizeof (GLfloat), &v_vertices[0]); + w = ((gfloat) pad_width / (gfloat) out_width); + h = ((gfloat) pad_height / (gfloat) out_height); - gl->VertexAttribPointer (attr_texture_loc, 2, GL_FLOAT, - GL_FALSE, 5 * sizeof (GLfloat), &v_vertices[3]); + /* top-left */ + v_vertices[0] = v_vertices[15] = + 2.0f * (gfloat) pad->xpos / (gfloat) out_width - 1.0f; + /* bottom-left */ + v_vertices[1] = v_vertices[6] = + 2.0f * (gfloat) pad->ypos / (gfloat) out_height - 1.0f; + /* top-right */ + v_vertices[5] = v_vertices[10] = v_vertices[0] + 2.0f * w; + /* bottom-right */ + v_vertices[11] = v_vertices[16] = v_vertices[1] + 2.0f * h; + GST_TRACE ("processing texture:%u dimensions:%ux%u, at %f,%f %fx%f with " + "alpha:%f", in_tex, in_width, in_height, v_vertices[0], v_vertices[1], + v_vertices[5], v_vertices[11], pad->alpha); - gl->EnableVertexAttribArray (attr_position_loc); - gl->EnableVertexAttribArray (attr_texture_loc); + if (!pad->vertex_buffer) + gl->GenBuffers (1, &pad->vertex_buffer); + + gl->BindBuffer (GL_ARRAY_BUFFER, pad->vertex_buffer); + + gl->BufferData (GL_ARRAY_BUFFER, 4 * 5 * sizeof (GLfloat), v_vertices, + GL_STATIC_DRAW); + } else { + gl->BindBuffer (GL_ARRAY_BUFFER, pad->vertex_buffer); + } gl->BlendFunc (GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA); gl->BlendEquation (GL_FUNC_ADD); @@ -662,6 +725,15 @@ gst_gl_video_mixer_callback (gpointer stuff) gst_gl_shader_set_uniform_1i (video_mixer->shader, "texture", 0); gst_gl_shader_set_uniform_1f (video_mixer->shader, "alpha", pad->alpha); + gl->EnableVertexAttribArray (attr_position_loc); + gl->EnableVertexAttribArray (attr_texture_loc); + + gl->VertexAttribPointer (attr_position_loc, 3, GL_FLOAT, + GL_FALSE, 5 * sizeof (GLfloat), (void *) 0); + + gl->VertexAttribPointer (attr_texture_loc, 2, GL_FLOAT, + GL_FALSE, 5 * sizeof (GLfloat), (void *) (3 * sizeof (GLfloat))); + gl->DrawElements (GL_TRIANGLES, 6, GL_UNSIGNED_SHORT, indices); ++count; @@ -670,6 +742,11 @@ gst_gl_video_mixer_callback (gpointer stuff) gl->DisableVertexAttribArray (attr_position_loc); gl->DisableVertexAttribArray (attr_texture_loc); + if (gl->GenVertexArrays) + gl->BindVertexArray (0); + else + gl->BindBuffer (GL_ARRAY_BUFFER, 0); + gl->BindTexture (GL_TEXTURE_2D, 0); gl->Disable (GL_BLEND); diff --git a/ext/gl/gstglvideomixer.h b/ext/gl/gstglvideomixer.h index d7dd070737..bc1cc7e7b8 100644 --- a/ext/gl/gstglvideomixer.h +++ b/ext/gl/gstglvideomixer.h @@ -63,6 +63,9 @@ struct _GstGLVideoMixer GstGLShader *shader; GstGLShader *checker; GPtrArray *input_frames; + + GLuint vao; + GLuint checker_vbo; }; struct _GstGLVideoMixerClass From 8f86923b00c5625510ca0c0bfb1f200b863f4a1c Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Fri, 17 Oct 2014 15:22:24 +0200 Subject: [PATCH 089/381] gl: add a sync meta for synchronizing across GL contexts A context can create a GLsync object that can be waited on in order to ensure that GL resources created in one context are able to be used in another shared context without any chance of reading invalid data. This meta would be placed on buffers that are known to cross from one context to another. The receiving element would then wait on the sync object to ensure that the data to be used is complete. --- ext/gl/gstglmixer.c | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/ext/gl/gstglmixer.c b/ext/gl/gstglmixer.c index bd63c75a1f..e245bc29fe 100644 --- a/ext/gl/gstglmixer.c +++ b/ext/gl/gstglmixer.c @@ -223,6 +223,8 @@ gst_gl_mixer_propose_allocation (GstGLMixer * mix, /* we also support various metadata */ gst_query_add_allocation_meta (query, GST_VIDEO_META_API_TYPE, 0); + if (mix->context->gl_vtable->FenceSync) + gst_query_add_allocation_meta (query, GST_GL_SYNC_META_API_TYPE, 0); gl_apis = gst_gl_api_to_string (gst_gl_context_get_gl_api (mix->context)); platform = @@ -778,6 +780,7 @@ gst_gl_mixer_decide_allocation (GstGLMixer * mix, GstQuery * query) guint out_width, out_height; GstGLContext *other_context = NULL; GstVideoAggregator *vagg = GST_VIDEO_AGGREGATOR (mix); + gboolean same_downstream_gl_context = FALSE; if (!gst_gl_ensure_element_data (mix, &mix->display, &mix->other_context)) return FALSE; @@ -799,6 +802,7 @@ gst_gl_mixer_decide_allocation (GstGLMixer * mix, GstQuery * query) mix->context = context; if (old) gst_object_unref (old); + same_downstream_gl_context = TRUE; } else if (gst_structure_get (upload_meta_params, "gst.gl.context.handle", G_TYPE_POINTER, &handle, "gst.gl.context.type", G_TYPE_STRING, &type, "gst.gl.context.apis", G_TYPE_STRING, &apis, NULL) @@ -882,13 +886,23 @@ gst_gl_mixer_decide_allocation (GstGLMixer * mix, GstQuery * query) update_pool = FALSE; } - if (!pool) + if (!pool || (!same_downstream_gl_context + && gst_query_find_allocation_meta (query, GST_GL_SYNC_META_API_TYPE, + NULL) + && !gst_buffer_pool_has_option (pool, + GST_BUFFER_POOL_OPTION_GL_SYNC_META))) { + /* can't use this pool */ + if (pool) + gst_object_unref (pool); pool = gst_gl_buffer_pool_new (mix->context); - + } config = gst_buffer_pool_get_config (pool); - gst_buffer_pool_config_set_params (config, caps, size, min, max); + gst_buffer_pool_config_set_params (config, caps, size, min, max); gst_buffer_pool_config_add_option (config, GST_BUFFER_POOL_OPTION_VIDEO_META); + if (gst_query_find_allocation_meta (query, GST_GL_SYNC_META_API_TYPE, NULL)) + gst_buffer_pool_config_add_option (config, + GST_BUFFER_POOL_OPTION_GL_SYNC_META); gst_buffer_pool_config_add_option (config, GST_BUFFER_POOL_OPTION_VIDEO_GL_TEXTURE_UPLOAD_META); @@ -1075,6 +1089,7 @@ gst_gl_mixer_process_textures (GstGLMixer * mix, GstBuffer * outbuf) GstCapsFeatures *gl_features; GstVideoInfo gl_info; GstVideoFrame gl_frame; + GstGLSyncMeta *sync_meta; gst_video_info_set_format (&gl_info, GST_VIDEO_FORMAT_RGBA, @@ -1096,6 +1111,10 @@ gst_gl_mixer_process_textures (GstGLMixer * mix, GstBuffer * outbuf) gst_caps_unref (in_caps); } + sync_meta = gst_buffer_get_gl_sync_meta (vaggpad->buffer); + if (sync_meta) + gst_gl_sync_meta_wait (sync_meta); + if (!gst_gl_upload_perform_with_buffer (pad->upload, vaggpad->buffer, &gl_buf)) { ++array_index; @@ -1207,12 +1226,17 @@ gst_gl_mixer_aggregate_frames (GstVideoAggregator * vagg, GstBuffer * outbuf) gboolean res = FALSE; GstGLMixer *mix = GST_GL_MIXER (vagg); GstGLMixerClass *mix_class = GST_GL_MIXER_GET_CLASS (vagg); + GstGLSyncMeta *sync_meta; if (mix_class->process_buffers) res = gst_gl_mixer_process_buffers (mix, outbuf); else if (mix_class->process_textures) res = gst_gl_mixer_process_textures (mix, outbuf); + sync_meta = gst_buffer_get_gl_sync_meta (outbuf); + if (sync_meta) + gst_gl_sync_meta_set_sync_point (sync_meta, mix->context); + return res ? GST_FLOW_OK : GST_FLOW_ERROR; } From c90f7001b131cc934eca0e30ce02ce6d75046a4a Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Thu, 27 Nov 2014 21:05:45 +1100 Subject: [PATCH 090/381] gldisplay: implement runtime GL api filtering Needed so that the pipeline/application can limit the choice of GL api to what it supports --- ext/gl/gstglmixer.c | 24 ++++++++++++++++++++++-- ext/gl/gstglmixer.h | 1 + ext/gl/gstglmosaic.c | 2 ++ ext/gl/gstglvideomixer.c | 3 +++ 4 files changed, 28 insertions(+), 2 deletions(-) diff --git a/ext/gl/gstglmixer.c b/ext/gl/gstglmixer.c index e245bc29fe..66b069ef76 100644 --- a/ext/gl/gstglmixer.c +++ b/ext/gl/gstglmixer.c @@ -152,6 +152,7 @@ static gboolean gst_gl_mixer_propose_allocation (GstGLMixer * mix, GstQuery * decide_query, GstQuery * query) { + GstGLMixerClass *mix_class = GST_GL_MIXER_GET_CLASS (mix); GstBufferPool *pool; GstStructure *config; GstCaps *caps; @@ -192,6 +193,8 @@ gst_gl_mixer_propose_allocation (GstGLMixer * mix, if (!gst_gl_ensure_element_data (mix, &mix->display, &mix->other_context)) return FALSE; + gst_gl_display_filter_gl_api (mix->display, mix_class->supported_gl_api); + if (!mix->context) { mix->context = gst_gl_context_new (mix->display); if (!gst_gl_context_create (mix->context, mix->other_context, &error)) @@ -428,6 +431,7 @@ gst_gl_mixer_sink_query (GstAggregator * agg, GstAggregatorPad * bpad, { gboolean ret = FALSE; GstGLMixer *mix = GST_GL_MIXER (agg); + GstGLMixerClass *mix_class = GST_GL_MIXER_GET_CLASS (mix); GST_TRACE ("QUERY %" GST_PTR_FORMAT, query); @@ -485,6 +489,9 @@ gst_gl_mixer_sink_query (GstAggregator * agg, GstAggregatorPad * bpad, { ret = gst_gl_handle_context_query ((GstElement *) mix, query, &mix->display, &mix->other_context); + if (mix->display) + gst_gl_display_filter_gl_api (mix->display, + mix_class->supported_gl_api); break; } default: @@ -615,7 +622,7 @@ gst_gl_mixer_class_init (GstGLMixerClass * klass) g_type_class_ref (GST_TYPE_GL_MIXER_PAD); klass->set_caps = NULL; - + klass->supported_gl_api = GST_GL_API_ANY; } static void @@ -661,19 +668,26 @@ static void gst_gl_mixer_set_context (GstElement * element, GstContext * context) { GstGLMixer *mix = GST_GL_MIXER (element); + GstGLMixerClass *mix_class = GST_GL_MIXER_GET_CLASS (mix); gst_gl_handle_set_context (element, context, &mix->display, &mix->other_context); + + if (mix->display) + gst_gl_display_filter_gl_api (mix->display, mix_class->supported_gl_api); } static gboolean gst_gl_mixer_activate (GstGLMixer * mix, gboolean active) { + GstGLMixerClass *mix_class = GST_GL_MIXER_GET_CLASS (mix); gboolean result = TRUE; if (active) { if (!gst_gl_ensure_element_data (mix, &mix->display, &mix->other_context)) - result = FALSE; + return FALSE; + + gst_gl_display_filter_gl_api (mix->display, mix_class->supported_gl_api); } return result; @@ -729,12 +743,16 @@ gst_gl_mixer_src_query (GstAggregator * agg, GstQuery * query) { gboolean res = FALSE; GstGLMixer *mix = GST_GL_MIXER (agg); + GstGLMixerClass *mix_class = GST_GL_MIXER_GET_CLASS (mix); switch (GST_QUERY_TYPE (query)) { case GST_QUERY_CONTEXT: { res = gst_gl_handle_context_query ((GstElement *) mix, query, &mix->display, &mix->other_context); + if (mix->display) + gst_gl_display_filter_gl_api (mix->display, + mix_class->supported_gl_api); break; } case GST_QUERY_CAPS: @@ -785,6 +803,8 @@ gst_gl_mixer_decide_allocation (GstGLMixer * mix, GstQuery * query) if (!gst_gl_ensure_element_data (mix, &mix->display, &mix->other_context)) return FALSE; + gst_gl_display_filter_gl_api (mix->display, mixer_class->supported_gl_api); + if (gst_query_find_allocation_meta (query, GST_VIDEO_GL_TEXTURE_UPLOAD_META_API_TYPE, &idx)) { GstGLContext *context; diff --git a/ext/gl/gstglmixer.h b/ext/gl/gstglmixer.h index f25c30a32f..4a92d54713 100644 --- a/ext/gl/gstglmixer.h +++ b/ext/gl/gstglmixer.h @@ -76,6 +76,7 @@ struct _GstGLMixer struct _GstGLMixerClass { GstVideoAggregatorClass parent_class; + GstGLAPI supported_gl_api; GstGLMixerSetCaps set_caps; GstGLMixerReset reset; diff --git a/ext/gl/gstglmosaic.c b/ext/gl/gstglmosaic.c index 1274a3dcec..e8daef9f6d 100644 --- a/ext/gl/gstglmosaic.c +++ b/ext/gl/gstglmosaic.c @@ -134,6 +134,8 @@ gst_gl_mosaic_class_init (GstGLMosaicClass * klass) GST_GL_MIXER_CLASS (klass)->set_caps = gst_gl_mosaic_init_shader; GST_GL_MIXER_CLASS (klass)->reset = gst_gl_mosaic_reset; GST_GL_MIXER_CLASS (klass)->process_textures = gst_gl_mosaic_process_textures; + + GST_GL_MIXER_CLASS (klass)->supported_gl_api = GST_GL_API_OPENGL; } static void diff --git a/ext/gl/gstglvideomixer.c b/ext/gl/gstglvideomixer.c index d73bf97150..f16f6d325f 100644 --- a/ext/gl/gstglvideomixer.c +++ b/ext/gl/gstglvideomixer.c @@ -345,6 +345,9 @@ gst_gl_video_mixer_class_init (GstGLVideoMixerClass * klass) vagg_class->update_caps = _update_caps; agg_class->sinkpads_type = GST_TYPE_GL_VIDEO_MIXER_PAD; + + GST_GL_MIXER_CLASS (klass)->supported_gl_api = + GST_GL_API_OPENGL | GST_GL_API_OPENGL3 | GST_GL_API_GLES2; } static void From 6354e0235a68de265e33aa98153b32646f791b57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Fri, 28 Nov 2014 10:22:44 +0100 Subject: [PATCH 091/381] videoaggregator: Failure to map a video frame is not just a warning --- gst-libs/gst/video/gstvideoaggregator.c | 1 + 1 file changed, 1 insertion(+) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index 91463b680c..e1c4c512fa 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -229,6 +229,7 @@ gst_video_aggregator_pad_prepare_frame (GstVideoAggregatorPad * pad, if (!gst_video_frame_map (frame, &pad->buffer_vinfo, pad->buffer, GST_MAP_READ)) { GST_WARNING_OBJECT (vagg, "Could not map input buffer"); + return FALSE; } if (pad->priv->convert) { From b212aca601602304eaebf359fdd4323f10387b0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Fri, 28 Nov 2014 10:23:55 +0100 Subject: [PATCH 092/381] compositor: Failure to map a video frame is not just a warning Also add some warning debug output if mapping a output buffer fails --- gst/compositor/compositor.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/gst/compositor/compositor.c b/gst/compositor/compositor.c index 1befdcc39e..04d01a14cc 100644 --- a/gst/compositor/compositor.c +++ b/gst/compositor/compositor.c @@ -283,6 +283,7 @@ gst_compositor_pad_prepare_frame (GstVideoAggregatorPad * pad, if (!gst_video_frame_map (frame, &pad->buffer_vinfo, pad->buffer, GST_MAP_READ)) { GST_WARNING_OBJECT (vagg, "Could not map input buffer"); + return FALSE; } if (cpad->width > 0) @@ -762,7 +763,7 @@ gst_compositor_aggregate_frames (GstVideoAggregator * vagg, GstBuffer * outbuf) GstVideoFrame out_frame, *outframe; if (!gst_video_frame_map (&out_frame, &vagg->info, outbuf, GST_MAP_WRITE)) { - + GST_WARNING_OBJECT (vagg, "Could not map output buffer"); return GST_FLOW_ERROR; } From 85febbbafbe48f82d62cec09b119560142ed319f Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Thu, 11 Dec 2014 18:15:02 +1100 Subject: [PATCH 093/381] gl: fixup vao and vbo usage for legacy GL --- ext/gl/gstglvideomixer.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ext/gl/gstglvideomixer.c b/ext/gl/gstglvideomixer.c index f16f6d325f..d673daee32 100644 --- a/ext/gl/gstglvideomixer.c +++ b/ext/gl/gstglvideomixer.c @@ -747,9 +747,8 @@ gst_gl_video_mixer_callback (gpointer stuff) if (gl->GenVertexArrays) gl->BindVertexArray (0); - else - gl->BindBuffer (GL_ARRAY_BUFFER, 0); + gl->BindBuffer (GL_ARRAY_BUFFER, 0); gl->BindTexture (GL_TEXTURE_2D, 0); gl->Disable (GL_BLEND); From e8fda3a28a9c4d6141407e022b5b33645c3f9941 Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Thu, 27 Nov 2014 20:48:24 +0100 Subject: [PATCH 094/381] videoaggregator: Hide some more fields from the API + Add some documentation --- gst-libs/gst/video/gstvideoaggregator.c | 37 +++++++++++++------------ 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index e1c4c512fa..6a6cfcb253 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -69,6 +69,9 @@ struct _GstVideoAggregatorPadPrivate /* caps used for conversion if needed */ GstVideoInfo conversion_info; GstBuffer *converted_buffer; + + GstClockTime start_time; + GstClockTime end_time; }; G_DEFINE_TYPE (GstVideoAggregatorPad, gst_videoaggregator_pad, @@ -129,8 +132,8 @@ _flush_pad (GstAggregatorPad * aggpad, GstAggregator * aggregator) gst_videoaggregator_reset_qos (vagg); gst_buffer_replace (&pad->buffer, NULL); - pad->start_time = -1; - pad->end_time = -1; + pad->priv->start_time = -1; + pad->priv->end_time = -1; return TRUE; } @@ -914,8 +917,8 @@ gst_videoaggregator_reset (GstVideoAggregator * vagg) GstVideoAggregatorPad *p = l->data; gst_buffer_replace (&p->buffer, NULL); - p->start_time = -1; - p->end_time = -1; + p->priv->start_time = -1; + p->priv->end_time = -1; gst_video_info_init (&p->info); } @@ -1022,7 +1025,7 @@ gst_videoaggregator_fill_queues (GstVideoAggregator * vagg, end_time *= ABS (agg->segment.rate); } - if (pad->end_time != -1 && pad->end_time > end_time + if (pad->priv->end_time != -1 && pad->priv->end_time > end_time && !bpad->unresponsive) { GST_DEBUG_OBJECT (pad, "Buffer from the past, dropping"); gst_buffer_unref (buf); @@ -1038,8 +1041,8 @@ gst_videoaggregator_fill_queues (GstVideoAggregator * vagg, GST_TIME_ARGS (start_time)); gst_buffer_replace (&pad->buffer, buf); pad->buffer_vinfo = *vinfo; - pad->start_time = start_time; - pad->end_time = end_time; + pad->priv->start_time = start_time; + pad->priv->end_time = end_time; gst_buffer_unref (buf); buf = gst_aggregator_pad_steal_buffer (bpad); @@ -1065,9 +1068,9 @@ gst_videoaggregator_fill_queues (GstVideoAggregator * vagg, continue; } } else { - if (!bpad->unresponsive && pad->end_time != -1) { - if (pad->end_time <= output_start_time) { - pad->start_time = pad->end_time = -1; + if (!bpad->unresponsive && pad->priv->end_time != -1) { + if (pad->priv->end_time <= output_start_time) { + pad->priv->start_time = pad->priv->end_time = -1; if (is_eos) { GST_DEBUG ("I just need more data"); need_more_data = TRUE; @@ -1517,7 +1520,7 @@ gst_videoaggregator_sink_clip (GstAggregator * agg, end_time *= ABS (agg->segment.rate); } - if (bpad->buffer != NULL && end_time < pad->end_time) { + if (bpad->buffer != NULL && end_time < pad->priv->end_time) { gst_buffer_unref (buf); *outbuf = NULL; return GST_FLOW_OK; @@ -1543,12 +1546,12 @@ gst_videoaggregator_flush (GstAggregator * agg) /* Convert to the output segment rate */ if (ABS (agg->segment.rate) != abs_rate) { if (ABS (agg->segment.rate) != 1.0 && p->buffer) { - p->start_time /= ABS (agg->segment.rate); - p->end_time /= ABS (agg->segment.rate); + p->priv->start_time /= ABS (agg->segment.rate); + p->priv->end_time /= ABS (agg->segment.rate); } if (abs_rate != 1.0 && p->buffer) { - p->start_time *= abs_rate; - p->end_time *= abs_rate; + p->priv->start_time *= abs_rate; + p->priv->end_time *= abs_rate; } } } @@ -1650,8 +1653,8 @@ gst_videoaggregator_request_new_pad (GstElement * element, GST_OBJECT_LOCK (vagg); vaggpad->zorder = GST_ELEMENT (vagg)->numsinkpads; - vaggpad->start_time = -1; - vaggpad->end_time = -1; + vaggpad->priv->start_time = -1; + vaggpad->priv->end_time = -1; element->sinkpads = g_list_sort (element->sinkpads, (GCompareFunc) pad_zorder_compare); GST_OBJECT_UNLOCK (vagg); From 8fe8510d80518a667b0813d09111530b001051ba Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Mon, 8 Dec 2014 15:18:25 +1100 Subject: [PATCH 095/381] videoaggregator: always try to use newer buffers instead of dropping them for being too old. This ensures that the newest buffer is always used for rendering --- gst-libs/gst/video/gstvideoaggregator.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index 6a6cfcb253..91b9276767 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -1055,9 +1055,10 @@ gst_videoaggregator_fill_queues (GstVideoAggregator * vagg, gst_buffer_unref (buf); eos = FALSE; } else { + gst_buffer_replace (&pad->buffer, buf); GST_DEBUG_OBJECT (pad, - "Too old buffer -- dropping start %" GST_TIME_FORMAT " out end %" - GST_TIME_FORMAT, GST_TIME_ARGS (start_time), + "replacing old buffer with a newer buffer, start %" GST_TIME_FORMAT + " out end %" GST_TIME_FORMAT, GST_TIME_ARGS (start_time), GST_TIME_ARGS (output_end_time)); gst_buffer_unref (buf); buf = gst_aggregator_pad_steal_buffer (bpad); From e354149436570850fd0ada6c0356085729b63c22 Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Fri, 5 Dec 2014 18:19:54 +1100 Subject: [PATCH 096/381] aggregator: make the src pad task drive the pipeline for live pipelines This removes the uses of GAsyncQueue and replaces it with explicit GMutex, GCond and wakeup count which is used for the non-live case. For live pipelines, the aggregator waits on the clock until either data arrives on all sink pads or the expected output buffer time arrives plus the timeout/latency at which time, the subclass produces a buffer. https://bugzilla.gnome.org/show_bug.cgi?id=741146 --- gst-libs/gst/video/gstvideoaggregator.c | 39 +++++++++++++++++-------- 1 file changed, 27 insertions(+), 12 deletions(-) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index 91b9276767..a711beea8c 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -1025,13 +1025,17 @@ gst_videoaggregator_fill_queues (GstVideoAggregator * vagg, end_time *= ABS (agg->segment.rate); } - if (pad->priv->end_time != -1 && pad->priv->end_time > end_time - && !bpad->unresponsive) { + GST_TRACE_OBJECT (pad, "dealing with buffer %p start %" GST_TIME_FORMAT + " end %" GST_TIME_FORMAT " out start %" GST_TIME_FORMAT + " out end %" GST_TIME_FORMAT, buf, GST_TIME_ARGS (start_time), + GST_TIME_ARGS (end_time), GST_TIME_ARGS (output_start_time), + GST_TIME_ARGS (output_end_time)); + + if (pad->priv->end_time != -1 && pad->priv->end_time > end_time) { GST_DEBUG_OBJECT (pad, "Buffer from the past, dropping"); gst_buffer_unref (buf); buf = gst_aggregator_pad_steal_buffer (bpad); gst_buffer_unref (buf); - need_more_data = TRUE; continue; } @@ -1069,7 +1073,7 @@ gst_videoaggregator_fill_queues (GstVideoAggregator * vagg, continue; } } else { - if (!bpad->unresponsive && pad->priv->end_time != -1) { + if (pad->priv->end_time != -1) { if (pad->priv->end_time <= output_start_time) { pad->priv->start_time = pad->priv->end_time = -1; if (is_eos) { @@ -1216,6 +1220,15 @@ gst_videoaggregator_do_qos (GstVideoAggregator * vagg, GstClockTime timestamp) return jitter; } +static GstClockTime +gst_videoaggregator_get_next_time (GstAggregator * agg) +{ + if (agg->segment.position == -1) + return agg->segment.start; + else + return agg->segment.position; +} + static GstFlowReturn gst_videoaggregator_aggregate (GstAggregator * agg) { @@ -1240,15 +1253,16 @@ gst_videoaggregator_aggregate (GstAggregator * agg) } } - if (agg->segment.position == -1) - output_start_time = agg->segment.start; - else - output_start_time = agg->segment.position; + output_start_time = gst_videoaggregator_get_next_time (agg); - output_end_time = - vagg->priv->ts_offset + gst_util_uint64_scale_round (vagg->priv->nframes + - 1, GST_SECOND * GST_VIDEO_INFO_FPS_D (&vagg->info), - GST_VIDEO_INFO_FPS_N (&vagg->info)) + agg->segment.start; + if (GST_VIDEO_INFO_FPS_N (&vagg->info) == 0) + output_end_time = -1; + else + output_end_time = + vagg->priv->ts_offset + + gst_util_uint64_scale_round (vagg->priv->nframes + 1, + GST_SECOND * GST_VIDEO_INFO_FPS_D (&vagg->info), + GST_VIDEO_INFO_FPS_N (&vagg->info)) + agg->segment.start; if (agg->segment.stop != -1) output_end_time = MIN (output_end_time, agg->segment.stop); @@ -1895,6 +1909,7 @@ gst_videoaggregator_class_init (GstVideoAggregatorClass * klass) agg_class->aggregate = gst_videoaggregator_aggregate; agg_class->src_event = gst_videoaggregator_src_event; agg_class->src_query = gst_videoaggregator_src_query; + agg_class->get_next_time = gst_videoaggregator_get_next_time; klass->find_best_format = gst_videoaggreagator_find_best_format; klass->get_output_buffer = gst_videoaggregator_get_output_buffer; From 26d1a22e903a62592b850db0c64badff7ca510cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Wed, 17 Dec 2014 17:54:09 +0100 Subject: [PATCH 097/381] aggregator: Add a timeout parameter to ::aggregate() When this is TRUE, we really have to produce output. This happens in live mixing mode when we have to output something for the current time, no matter if we have enough input or not. --- gst-libs/gst/video/gstvideoaggregator.c | 4 ++-- gst/compositor/compositor.c | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index a711beea8c..898de598f2 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -1230,7 +1230,7 @@ gst_videoaggregator_get_next_time (GstAggregator * agg) } static GstFlowReturn -gst_videoaggregator_aggregate (GstAggregator * agg) +gst_videoaggregator_aggregate (GstAggregator * agg, gboolean timeout) { GstFlowReturn ret; GstVideoAggregator *vagg = GST_VIDEO_AGGREGATOR (agg); @@ -1275,7 +1275,7 @@ gst_videoaggregator_aggregate (GstAggregator * agg) output_end_time); } - if (res == GST_FLOW_NEEDS_DATA) { + if (res == GST_FLOW_NEEDS_DATA && !timeout) { GST_DEBUG_OBJECT (vagg, "Need more data for decisions"); ret = GST_FLOW_OK; goto done; diff --git a/gst/compositor/compositor.c b/gst/compositor/compositor.c index 04d01a14cc..c5d09011c5 100644 --- a/gst/compositor/compositor.c +++ b/gst/compositor/compositor.c @@ -280,6 +280,9 @@ gst_compositor_pad_prepare_frame (GstVideoAggregatorPad * pad, static GstAllocationParams params = { 0, 15, 0, 0, }; gint width, height; + if (!pad->buffer) + return TRUE; + if (!gst_video_frame_map (frame, &pad->buffer_vinfo, pad->buffer, GST_MAP_READ)) { GST_WARNING_OBJECT (vagg, "Could not map input buffer"); From 6f0eb92d7245e875edf122ef105cf5e185996380 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Wed, 17 Dec 2014 19:51:32 +0100 Subject: [PATCH 098/381] aggregator: Add function to allow subclasses to set their own latency For audiomixer this is one blocksize, for videoaggregator this should be the duration of one output frame. --- gst-libs/gst/video/gstvideoaggregator.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index 898de598f2..a8c004d0fe 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -597,6 +597,9 @@ gst_videoaggregator_src_setcaps (GstVideoAggregator * vagg, GstCaps * caps) GST_VIDEO_AGGREGATOR_UNLOCK (vagg); gst_aggregator_set_src_caps (agg, caps); + gst_aggregator_set_latency (agg, gst_util_uint64_scale (GST_SECOND, + GST_VIDEO_INFO_FPS_D (&info), GST_VIDEO_INFO_FPS_N (&info)), + GST_CLOCK_TIME_NONE); GST_VIDEO_AGGREGATOR_LOCK (vagg); } From a64445e543fef4e2dc51f4505fbda4c01891a94c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Thu, 18 Dec 2014 11:48:37 +0100 Subject: [PATCH 099/381] compositor: Fix memory leak when no buffer is available for a pad currently CID 1258718 --- gst/compositor/compositor.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/gst/compositor/compositor.c b/gst/compositor/compositor.c index c5d09011c5..06d96bfce5 100644 --- a/gst/compositor/compositor.c +++ b/gst/compositor/compositor.c @@ -276,13 +276,15 @@ gst_compositor_pad_prepare_frame (GstVideoAggregatorPad * pad, guint outsize; GstVideoFrame *converted_frame; GstBuffer *converted_buf = NULL; - GstVideoFrame *frame = g_slice_new0 (GstVideoFrame); + GstVideoFrame *frame; static GstAllocationParams params = { 0, 15, 0, 0, }; gint width, height; if (!pad->buffer) return TRUE; + frame = g_slice_new0 (GstVideoFrame); + if (!gst_video_frame_map (frame, &pad->buffer_vinfo, pad->buffer, GST_MAP_READ)) { GST_WARNING_OBJECT (vagg, "Could not map input buffer"); From ccd24a357741524e6cc7e102d57804036e71a084 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Thu, 18 Dec 2014 22:03:04 +0100 Subject: [PATCH 100/381] videoaggregator: Make sure to always update the pad's videoinfo together with buffers Otherwise the videoinfo and the buffer content can go out of sync. --- gst-libs/gst/video/gstvideoaggregator.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index a8c004d0fe..9a6374a788 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -987,6 +987,8 @@ gst_videoaggregator_fill_queues (GstVideoAggregator * vagg, gst_buffer_unref (buf); buf = gst_aggregator_pad_steal_buffer (bpad); gst_buffer_replace (&pad->buffer, buf); + pad->buffer_vinfo = *vinfo; + /* FIXME: Set start_time and end_time to something here? */ gst_buffer_unref (buf); GST_DEBUG_OBJECT (pad, "buffer duration is -1"); continue; @@ -1063,6 +1065,9 @@ gst_videoaggregator_fill_queues (GstVideoAggregator * vagg, eos = FALSE; } else { gst_buffer_replace (&pad->buffer, buf); + pad->buffer_vinfo = *vinfo; + pad->priv->start_time = start_time; + pad->priv->end_time = end_time; GST_DEBUG_OBJECT (pad, "replacing old buffer with a newer buffer, start %" GST_TIME_FORMAT " out end %" GST_TIME_FORMAT, GST_TIME_ARGS (start_time), From 5c9472f78cd2837891f8986547b3edb68339b642 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Thu, 18 Dec 2014 22:04:38 +0100 Subject: [PATCH 101/381] videoaggregator: Don't try to map NULL buffers --- gst-libs/gst/video/gstvideoaggregator.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index 9a6374a788..023c3926b6 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -226,9 +226,14 @@ gst_video_aggregator_pad_prepare_frame (GstVideoAggregatorPad * pad, guint outsize; GstVideoFrame *converted_frame; GstBuffer *converted_buf = NULL; - GstVideoFrame *frame = g_slice_new0 (GstVideoFrame); + GstVideoFrame *frame; static GstAllocationParams params = { 0, 15, 0, 0, }; + if (!pad->buffer) + return TRUE; + + frame = g_slice_new0 (GstVideoFrame); + if (!gst_video_frame_map (frame, &pad->buffer_vinfo, pad->buffer, GST_MAP_READ)) { GST_WARNING_OBJECT (vagg, "Could not map input buffer"); From b9312fe9a2625fc77f6316693fd68f9edda97d61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Mon, 22 Dec 2014 22:11:30 +0100 Subject: [PATCH 102/381] videoaggregator: Use the src query implementation of aggregator as the default case --- gst-libs/gst/video/gstvideoaggregator.c | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index 023c3926b6..e5b4904406 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -1464,17 +1464,11 @@ gst_videoaggregator_src_query (GstAggregator * agg, GstQuery * query) case GST_QUERY_DURATION: res = gst_videoaggregator_query_duration (vagg, query); break; - case GST_QUERY_LATENCY: - case GST_QUERY_CAPS: + default: res = GST_AGGREGATOR_CLASS (gst_videoaggregator_parent_class)->src_query (agg, query); break; - default: - /* FIXME, needs a custom query handler because we have multiple - * sinkpads */ - res = FALSE; - break; } return res; } From a27431c6808301a5515ece931e522dfe2ffcf1ba Mon Sep 17 00:00:00 2001 From: Nirbheek Chauhan Date: Sat, 27 Dec 2014 15:49:47 +0530 Subject: [PATCH 103/381] compositor: Document the pad properties --- gst/compositor/compositor.c | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/gst/compositor/compositor.c b/gst/compositor/compositor.c index 06d96bfce5..ba08556590 100644 --- a/gst/compositor/compositor.c +++ b/gst/compositor/compositor.c @@ -31,7 +31,31 @@ * Compositor will do colorspace conversion. * * Individual parameters for each input stream can be configured on the - * #GstCompositorPad. + * #GstCompositorPad: + * + * + * + * "xpos": The x-coordinate position of the top-left corner of the picture + * (#gint) + * + * + * "ypos": The y-coordinate position of the top-left corner of the picture + * (#gint) + * + * + * "width": The width of the picture; the input will be scaled if necessary + * (#gint) + * + * + * "height": The height of the picture; the input will be scaled if necessary + * (#gint) + * + * + * "alpha": The transparency of the picture; between 0.0 and 1.0. The blending + * is a simple copy when fully-transparent (0.0) and fully-opaque (1.0). + * (#gdouble) + * + * * * * Sample pipelines From 6351c0becc51be144e037adf1c79e29bc6fdf0e1 Mon Sep 17 00:00:00 2001 From: Nirbheek Chauhan Date: Sat, 27 Dec 2014 15:50:17 +0530 Subject: [PATCH 104/381] glvideomixer: Point to compositor for the pad properties documentation --- ext/gl/gstglvideomixer.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ext/gl/gstglvideomixer.c b/ext/gl/gstglvideomixer.c index d673daee32..c163bf3a1b 100644 --- a/ext/gl/gstglvideomixer.c +++ b/ext/gl/gstglvideomixer.c @@ -22,7 +22,8 @@ * SECTION:element-glvideomixer * * Composites a number of streams into a single output scene using OpenGL in - * a similar fashion to compositor and videomixer. + * a similar fashion to compositor and videomixer. See the compositor plugin + * for documentation about the #GstGLVideoMixerPad properties. * * * Examples From 7d22da9510e1fb62e0cf6b40d1bec39e27ca5166 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim-Philipp=20M=C3=BCller?= Date: Sun, 28 Dec 2014 01:13:33 +0000 Subject: [PATCH 105/381] aggregator: make padding larger Esp. the class structures, can't have enough spare space for virtual functions. --- gst-libs/gst/video/gstvideoaggregator.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gst-libs/gst/video/gstvideoaggregator.h b/gst-libs/gst/video/gstvideoaggregator.h index 60733ed2c9..2be7eb6dc0 100644 --- a/gst-libs/gst/video/gstvideoaggregator.h +++ b/gst-libs/gst/video/gstvideoaggregator.h @@ -65,7 +65,7 @@ struct _GstVideoAggregator /* < private > */ GstVideoAggregatorPrivate *priv; - gpointer _gst_reserved[GST_PADDING]; + gpointer _gst_reserved[GST_PADDING_LARGE]; }; /** @@ -106,7 +106,7 @@ struct _GstVideoAggregatorClass GstVideoInfo * best_info, gboolean * at_least_one_alpha); /* < private > */ - gpointer _gst_reserved[GST_PADDING]; + gpointer _gst_reserved[GST_PADDING_LARGE]; }; GType gst_videoaggregator_get_type (void); From b8e97659fe9a97d44bbab62f1a535a5a6671dfec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim-Philipp=20M=C3=BCller?= Date: Tue, 30 Dec 2014 17:56:35 +0000 Subject: [PATCH 106/381] videoaggregator: update for aggregator start/stop vfunc change --- gst-libs/gst/video/gstvideoaggregator.c | 6 ------ 1 file changed, 6 deletions(-) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index e5b4904406..e850b6f7bb 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -1635,9 +1635,6 @@ gst_videoaggregator_start (GstAggregator * agg) { GstVideoAggregator *vagg = GST_VIDEO_AGGREGATOR (agg); - if (!GST_AGGREGATOR_CLASS (gst_videoaggregator_parent_class)->start (agg)) - return FALSE; - gst_caps_replace (&vagg->priv->current_caps, NULL); return TRUE; @@ -1648,9 +1645,6 @@ gst_videoaggregator_stop (GstAggregator * agg) { GstVideoAggregator *vagg = GST_VIDEO_AGGREGATOR (agg); - if (!GST_AGGREGATOR_CLASS (gst_videoaggregator_parent_class)->stop (agg)) - return FALSE; - gst_videoaggregator_reset (vagg); return TRUE; From 6ad57a882a3674321f6b617aef9bfa237ebcc40f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim-Philipp=20M=C3=BCller?= Date: Tue, 30 Dec 2014 19:26:47 +0000 Subject: [PATCH 107/381] glmixer: update for aggregator start/stop vfunc change --- ext/gl/gstglmixer.c | 6 ------ 1 file changed, 6 deletions(-) diff --git a/ext/gl/gstglmixer.c b/ext/gl/gstglmixer.c index 66b069ef76..744106ac2b 100644 --- a/ext/gl/gstglmixer.c +++ b/ext/gl/gstglmixer.c @@ -1313,9 +1313,6 @@ gst_gl_mixer_start (GstAggregator * agg) GstGLMixer *mix = GST_GL_MIXER (agg); GstElement *element = GST_ELEMENT (agg); - if (!GST_AGGREGATOR_CLASS (parent_class)->start (agg)) - return FALSE; - GST_OBJECT_LOCK (mix); mix->array_buffers = g_ptr_array_new_full (element->numsinkpads, (GDestroyNotify) _free_glmixer_frame_data); @@ -1338,9 +1335,6 @@ gst_gl_mixer_stop (GstAggregator * agg) GstGLMixer *mix = GST_GL_MIXER (agg); GstGLMixerClass *mixer_class = GST_GL_MIXER_GET_CLASS (mix); - if (!GST_AGGREGATOR_CLASS (parent_class)->stop (agg)) - return FALSE; - GST_OBJECT_LOCK (agg); g_ptr_array_free (mix->frames, TRUE); mix->frames = NULL; From 13af5efb07fb5ccda07c2ee0c016e513b07e5ded Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim-Philipp=20M=C3=BCller?= Date: Tue, 30 Dec 2014 19:43:43 +0000 Subject: [PATCH 108/381] glmixer, glvideomixer: update for GstAggregatorPadForeachFunc change --- ext/gl/gstglmixer.c | 2 +- ext/gl/gstglvideomixer.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ext/gl/gstglmixer.c b/ext/gl/gstglmixer.c index 744106ac2b..68de445806 100644 --- a/ext/gl/gstglmixer.c +++ b/ext/gl/gstglmixer.c @@ -1288,7 +1288,7 @@ gst_gl_mixer_set_property (GObject * object, } static gboolean -_clean_upload (GstAggregator * agg, GstPad * aggpad, gpointer udata) +_clean_upload (GstAggregator * agg, GstAggregatorPad * aggpad, gpointer udata) { GstGLMixerPad *pad = GST_GL_MIXER_PAD (aggpad); diff --git a/ext/gl/gstglvideomixer.c b/ext/gl/gstglvideomixer.c index c163bf3a1b..8641d04e21 100644 --- a/ext/gl/gstglvideomixer.c +++ b/ext/gl/gstglvideomixer.c @@ -446,7 +446,7 @@ _update_caps (GstVideoAggregator * vagg, GstCaps * caps) } static gboolean -_reset_pad_gl (GstAggregator * agg, GstPad * aggpad, gpointer udata) +_reset_pad_gl (GstAggregator * agg, GstAggregatorPad * aggpad, gpointer udata) { const GstGLFuncs *gl = GST_GL_MIXER (agg)->context->gl_vtable; GstGLVideoMixerPad *pad = GST_GL_VIDEO_MIXER_PAD (aggpad); From 7d8b88ae3b5f91571d61569dd646b2fa23846d14 Mon Sep 17 00:00:00 2001 From: Stefan Sauer Date: Mon, 12 Jan 2015 21:25:14 +0100 Subject: [PATCH 109/381] gitignore: ignore more files --- gst/compositor/.gitignore | 1 + 1 file changed, 1 insertion(+) create mode 100644 gst/compositor/.gitignore diff --git a/gst/compositor/.gitignore b/gst/compositor/.gitignore new file mode 100644 index 0000000000..131b2eef14 --- /dev/null +++ b/gst/compositor/.gitignore @@ -0,0 +1 @@ +compositororc.h From e1f3214c6ce91d3c34366d30636d4df74a4fe4c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Tue, 13 Jan 2015 14:54:26 +0100 Subject: [PATCH 110/381] compositor: Don't do any conversions if the pad is completely transparent anyway --- gst/compositor/compositor.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/gst/compositor/compositor.c b/gst/compositor/compositor.c index ba08556590..d3fa3f07b8 100644 --- a/gst/compositor/compositor.c +++ b/gst/compositor/compositor.c @@ -390,7 +390,12 @@ gst_compositor_pad_prepare_frame (GstVideoAggregatorPad * pad, g_free (wanted_colorimetry); } - if (cpad->convert) { + if (cpad->alpha == 0.0) { + GST_DEBUG_OBJECT (vagg, "Pad has alpha 0.0, not converting frame"); + converted_frame = NULL; + gst_video_frame_unmap (frame); + g_slice_free (GstVideoFrame, frame); + } else if (cpad->convert) { gint converted_size; converted_frame = g_slice_new0 (GstVideoFrame); From bd04b3628061dee5e649a6019259e0bb9f53d3ad Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Wed, 14 Jan 2015 22:08:43 +1100 Subject: [PATCH 111/381] gl: split glcolorconvert usage from glupload the separation allows the transfer operation to occur in a separate thread/time which may increase performance in specific circumstances. --- ext/gl/gstglmixer.c | 55 ++++++++++++++++++++++++++++++++++++--------- 1 file changed, 44 insertions(+), 11 deletions(-) diff --git a/ext/gl/gstglmixer.c b/ext/gl/gstglmixer.c index 68de445806..13a9dead99 100644 --- a/ext/gl/gstglmixer.c +++ b/ext/gl/gstglmixer.c @@ -1104,7 +1104,7 @@ gst_gl_mixer_process_textures (GstGLMixer * mix, GstBuffer * outbuf) walk = g_list_next (walk); if (vaggpad->buffer != NULL) { - GstBuffer *gl_buf; + GstBuffer *uploaded_buf; GstCaps *gl_caps; GstCapsFeatures *gl_features; GstVideoInfo gl_info; @@ -1119,45 +1119,68 @@ gst_gl_mixer_process_textures (GstGLMixer * mix, GstBuffer * outbuf) gst_caps_features_from_string (GST_CAPS_FEATURE_MEMORY_GL_MEMORY); gl_caps = gst_video_info_to_caps (&gl_info); - gst_caps_set_features (gl_caps, 0, gl_features); + gst_caps_set_features (gl_caps, 0, gst_caps_features_copy (gl_features)); if (!pad->upload) { GstCaps *in_caps = gst_pad_get_current_caps (GST_PAD (pad)); + GstCaps *upload_caps = gst_caps_copy (in_caps); pad->upload = gst_gl_upload_new (mix->context); - gst_gl_upload_set_caps (pad->upload, in_caps, gl_caps); + gst_caps_set_features (upload_caps, 0, + gst_caps_features_copy (gl_features)); + gst_gl_upload_set_caps (pad->upload, in_caps, upload_caps); + if (!pad->convert) { + pad->convert = gst_gl_color_convert_new (mix->context); + + gst_gl_color_convert_set_caps (pad->convert, upload_caps, gl_caps); + } + + gst_caps_unref (upload_caps); gst_caps_unref (in_caps); } + gst_caps_features_free (gl_features); + gst_caps_unref (gl_caps); + sync_meta = gst_buffer_get_gl_sync_meta (vaggpad->buffer); if (sync_meta) gst_gl_sync_meta_wait (sync_meta); - if (!gst_gl_upload_perform_with_buffer (pad->upload, - vaggpad->buffer, &gl_buf)) { + if (gst_gl_upload_perform_with_buffer (pad->upload, + vaggpad->buffer, &uploaded_buf) != GST_GL_UPLOAD_DONE) { ++array_index; pad->mapped = FALSE; - gst_caps_unref (gl_caps); continue; } - if (!gst_video_frame_map (&gl_frame, &gl_info, gl_buf, + if (pad->gl_buffer) + gst_buffer_unref (pad->gl_buffer); + + if (!(pad->gl_buffer = + gst_gl_color_convert_perform (pad->convert, uploaded_buf))) { + ++array_index; + pad->mapped = FALSE; + gst_buffer_unref (uploaded_buf); + continue; + } + + if (!gst_video_frame_map (&gl_frame, &gl_info, pad->gl_buffer, GST_MAP_READ | GST_MAP_GL)) { ++array_index; pad->mapped = FALSE; - gst_buffer_unref (gl_buf); - gst_caps_unref (gl_caps); + gst_buffer_unref (uploaded_buf); + gst_buffer_unref (pad->gl_buffer); + pad->gl_buffer = NULL; continue; } pad->mapped = TRUE; frame->texture = *(guint *) gl_frame.data[0]; - gst_caps_unref (gl_caps); + gst_buffer_unref (uploaded_buf); gst_video_frame_unmap (&gl_frame); - gst_buffer_unref (gl_buf); } ++array_index; } @@ -1292,11 +1315,21 @@ _clean_upload (GstAggregator * agg, GstAggregatorPad * aggpad, gpointer udata) { GstGLMixerPad *pad = GST_GL_MIXER_PAD (aggpad); + if (pad->gl_buffer) { + gst_buffer_unref (pad->gl_buffer); + pad->gl_buffer = NULL; + } + if (pad->upload) { gst_object_unref (pad->upload); pad->upload = NULL; } + if (pad->convert) { + gst_object_unref (pad->convert); + pad->convert = NULL; + } + return TRUE; } From 8e2ca1b6a79b78ba17142484643aae85dd45f7c2 Mon Sep 17 00:00:00 2001 From: Nirbheek Chauhan Date: Wed, 14 Jan 2015 23:48:16 +0530 Subject: [PATCH 112/381] compositor: Document the zorder pad property from gstvideoaggregator --- gst/compositor/compositor.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/gst/compositor/compositor.c b/gst/compositor/compositor.c index d3fa3f07b8..ef71d841b3 100644 --- a/gst/compositor/compositor.c +++ b/gst/compositor/compositor.c @@ -55,6 +55,10 @@ * is a simple copy when fully-transparent (0.0) and fully-opaque (1.0). * (#gdouble) * + * + * "zorder": The z-order position of the picture in the composition; between + * 0 and 10000. (#guint) + * * * * From 95a98d1d05962bd94700478fbd7ca3c9a59ec629 Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Thu, 15 Jan 2015 09:47:45 +1100 Subject: [PATCH 113/381] glvideomixer: don't upload if alpha <= 0 Implemented using a upload_buffer vfunc within GstGLMixer allowing NULL uploaded buffers. --- ext/gl/gstglmixer.c | 162 +++++++++++++++++++++------------------ ext/gl/gstglvideomixer.c | 21 +++++ 2 files changed, 108 insertions(+), 75 deletions(-) diff --git a/ext/gl/gstglmixer.c b/ext/gl/gstglmixer.c index 13a9dead99..5fdcc3db93 100644 --- a/ext/gl/gstglmixer.c +++ b/ext/gl/gstglmixer.c @@ -54,6 +54,8 @@ static void gst_gl_mixer_pad_get_property (GObject * object, guint prop_id, static void gst_gl_mixer_pad_set_property (GObject * object, guint prop_id, const GValue * value, GParamSpec * pspec); static void gst_gl_mixer_pad_finalize (GObject * object); +static GstBuffer *_default_pad_upload_buffer (GstGLMixer * mix, + GstGLMixerFrameData * frame, GstBuffer * buffer); static void gst_gl_mixer_set_context (GstElement * element, GstContext * context); @@ -98,6 +100,8 @@ gst_gl_mixer_pad_class_init (GstGLMixerPadClass * klass) vaggpad_class->set_info = NULL; vaggpad_class->prepare_frame = NULL; vaggpad_class->clean_frame = NULL; + + klass->upload_buffer = _default_pad_upload_buffer; } static void @@ -1043,6 +1047,81 @@ no_decide_allocation: } } +static GstBuffer * +_default_pad_upload_buffer (GstGLMixer * mix, GstGLMixerFrameData * frame, + GstBuffer * buffer) +{ + GstVideoAggregatorPad *vaggpad = GST_VIDEO_AGGREGATOR_PAD (frame->pad); + GstGLMixerPad *pad = frame->pad; + GstBuffer *uploaded_buf, *gl_buffer; + GstCaps *gl_caps; + GstCapsFeatures *gl_features; + GstVideoInfo gl_info; + GstVideoFrame gl_frame; + GstGLSyncMeta *sync_meta; + + gst_video_info_set_format (&gl_info, + GST_VIDEO_FORMAT_RGBA, + GST_VIDEO_INFO_WIDTH (&vaggpad->info), + GST_VIDEO_INFO_HEIGHT (&vaggpad->info)); + gl_features = + gst_caps_features_from_string (GST_CAPS_FEATURE_MEMORY_GL_MEMORY); + + gl_caps = gst_video_info_to_caps (&gl_info); + gst_caps_set_features (gl_caps, 0, gst_caps_features_copy (gl_features)); + + if (!pad->upload) { + GstCaps *in_caps = gst_pad_get_current_caps (GST_PAD (pad)); + GstCaps *upload_caps = gst_caps_copy (in_caps); + + pad->upload = gst_gl_upload_new (mix->context); + + gst_caps_set_features (upload_caps, 0, + gst_caps_features_copy (gl_features)); + gst_gl_upload_set_caps (pad->upload, in_caps, upload_caps); + + if (!pad->convert) { + pad->convert = gst_gl_color_convert_new (mix->context); + + gst_gl_color_convert_set_caps (pad->convert, upload_caps, gl_caps); + } + + gst_caps_unref (upload_caps); + gst_caps_unref (in_caps); + } + + gst_caps_features_free (gl_features); + gst_caps_unref (gl_caps); + + sync_meta = gst_buffer_get_gl_sync_meta (vaggpad->buffer); + if (sync_meta) + gst_gl_sync_meta_wait (sync_meta); + + if (gst_gl_upload_perform_with_buffer (pad->upload, + vaggpad->buffer, &uploaded_buf) != GST_GL_UPLOAD_DONE) { + return NULL; + } + + if (!(gl_buffer = gst_gl_color_convert_perform (pad->convert, uploaded_buf))) { + gst_buffer_unref (uploaded_buf); + return NULL; + } + + if (!gst_video_frame_map (&gl_frame, &gl_info, gl_buffer, + GST_MAP_READ | GST_MAP_GL)) { + gst_buffer_unref (uploaded_buf); + gst_buffer_unref (gl_buffer); + return NULL; + } + + frame->texture = *(guint *) gl_frame.data[0]; + + gst_buffer_unref (uploaded_buf); + gst_video_frame_unmap (&gl_frame); + + return gl_buffer; +} + gboolean gst_gl_mixer_process_textures (GstGLMixer * mix, GstBuffer * outbuf) { @@ -1094,6 +1173,7 @@ gst_gl_mixer_process_textures (GstGLMixer * mix, GstBuffer * outbuf) mix->frames->pdata[i] = g_slice_new0 (GstGLMixerFrameData); while (walk) { GstGLMixerPad *pad = GST_GL_MIXER_PAD (walk->data); + GstGLMixerPadClass *pad_class = GST_GL_MIXER_PAD_GET_CLASS (pad); GstVideoAggregatorPad *vaggpad = walk->data; GstGLMixerFrameData *frame; @@ -1104,84 +1184,17 @@ gst_gl_mixer_process_textures (GstGLMixer * mix, GstBuffer * outbuf) walk = g_list_next (walk); if (vaggpad->buffer != NULL) { - GstBuffer *uploaded_buf; - GstCaps *gl_caps; - GstCapsFeatures *gl_features; - GstVideoInfo gl_info; - GstVideoFrame gl_frame; - GstGLSyncMeta *sync_meta; - - gst_video_info_set_format (&gl_info, - GST_VIDEO_FORMAT_RGBA, - GST_VIDEO_INFO_WIDTH (&vaggpad->info), - GST_VIDEO_INFO_HEIGHT (&vaggpad->info)); - gl_features = - gst_caps_features_from_string (GST_CAPS_FEATURE_MEMORY_GL_MEMORY); - - gl_caps = gst_video_info_to_caps (&gl_info); - gst_caps_set_features (gl_caps, 0, gst_caps_features_copy (gl_features)); - - if (!pad->upload) { - GstCaps *in_caps = gst_pad_get_current_caps (GST_PAD (pad)); - GstCaps *upload_caps = gst_caps_copy (in_caps); - - pad->upload = gst_gl_upload_new (mix->context); - - gst_caps_set_features (upload_caps, 0, - gst_caps_features_copy (gl_features)); - gst_gl_upload_set_caps (pad->upload, in_caps, upload_caps); - - if (!pad->convert) { - pad->convert = gst_gl_color_convert_new (mix->context); - - gst_gl_color_convert_set_caps (pad->convert, upload_caps, gl_caps); - } - - gst_caps_unref (upload_caps); - gst_caps_unref (in_caps); - } - - gst_caps_features_free (gl_features); - gst_caps_unref (gl_caps); - - sync_meta = gst_buffer_get_gl_sync_meta (vaggpad->buffer); - if (sync_meta) - gst_gl_sync_meta_wait (sync_meta); - - if (gst_gl_upload_perform_with_buffer (pad->upload, - vaggpad->buffer, &uploaded_buf) != GST_GL_UPLOAD_DONE) { - ++array_index; - pad->mapped = FALSE; - continue; - } + g_assert (pad_class->upload_buffer); if (pad->gl_buffer) gst_buffer_unref (pad->gl_buffer); + pad->gl_buffer = pad_class->upload_buffer (mix, frame, vaggpad->buffer); - if (!(pad->gl_buffer = - gst_gl_color_convert_perform (pad->convert, uploaded_buf))) { - ++array_index; - pad->mapped = FALSE; - gst_buffer_unref (uploaded_buf); - continue; - } - - if (!gst_video_frame_map (&gl_frame, &gl_info, pad->gl_buffer, - GST_MAP_READ | GST_MAP_GL)) { - ++array_index; - pad->mapped = FALSE; - gst_buffer_unref (uploaded_buf); - gst_buffer_unref (pad->gl_buffer); - pad->gl_buffer = NULL; - continue; - } - pad->mapped = TRUE; - - frame->texture = *(guint *) gl_frame.data[0]; - - gst_buffer_unref (uploaded_buf); - gst_video_frame_unmap (&gl_frame); + GST_DEBUG_OBJECT (pad, + "uploaded buffer %" GST_PTR_FORMAT " from buffer %" GST_PTR_FORMAT, + pad->gl_buffer, vaggpad->buffer); } + ++array_index; } @@ -1217,10 +1230,9 @@ out: while (walk) { GstGLMixerPad *pad = GST_GL_MIXER_PAD (walk->data); - if (pad->mapped) + if (pad->upload) gst_gl_upload_release_buffer (pad->upload); - pad->mapped = FALSE; walk = g_list_next (walk); i++; } diff --git a/ext/gl/gstglvideomixer.c b/ext/gl/gstglvideomixer.c index 8641d04e21..c8aebcbac7 100644 --- a/ext/gl/gstglvideomixer.c +++ b/ext/gl/gstglvideomixer.c @@ -178,6 +178,8 @@ static void gst_gl_video_mixer_pad_set_property (GObject * object, guint prop_id, const GValue * value, GParamSpec * pspec); static void gst_gl_video_mixer_pad_get_property (GObject * object, guint prop_id, GValue * value, GParamSpec * pspec); +static GstBuffer *gst_gl_video_mixer_pad_upload_buffer (GstGLMixer * mix, + GstGLMixerFrameData * frame, GstBuffer * buffer); #define DEFAULT_PAD_XPOS 0 #define DEFAULT_PAD_YPOS 0 @@ -204,6 +206,7 @@ static void gst_gl_video_mixer_pad_class_init (GstGLVideoMixerPadClass * klass) { GObjectClass *gobject_class = (GObjectClass *) klass; + GstGLMixerPadClass *mix_pad_class = (GstGLMixerPadClass *) klass; gobject_class->set_property = gst_gl_video_mixer_pad_set_property; gobject_class->get_property = gst_gl_video_mixer_pad_get_property; @@ -228,6 +231,8 @@ gst_gl_video_mixer_pad_class_init (GstGLVideoMixerPadClass * klass) g_param_spec_double ("alpha", "Alpha", "Alpha of the picture", 0.0, 1.0, DEFAULT_PAD_ALPHA, G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); + + mix_pad_class->upload_buffer = gst_gl_video_mixer_pad_upload_buffer; } static void @@ -293,6 +298,22 @@ gst_gl_video_mixer_pad_set_property (GObject * object, guint prop_id, gst_object_unref (mix); } +static GstBuffer * +gst_gl_video_mixer_pad_upload_buffer (GstGLMixer * mix, + GstGLMixerFrameData * frame, GstBuffer * buffer) +{ + GstGLVideoMixerPad *pad = GST_GL_VIDEO_MIXER_PAD (frame->pad); + + if (pad->alpha > 0.0) { + return + GST_GL_MIXER_PAD_CLASS + (gst_gl_video_mixer_pad_parent_class)->upload_buffer (mix, frame, + buffer); + } + + return NULL; +} + #define GST_GL_TYPE_VIDEO_MIXER_BACKGROUND (gst_gl_video_mixer_background_get_type()) static GType gst_gl_video_mixer_background_get_type (void) From 6b2f6944551cb09d96c2ede4439bad1a5e96e7f5 Mon Sep 17 00:00:00 2001 From: Vincent Penquerc'h Date: Mon, 19 Jan 2015 12:37:23 +0000 Subject: [PATCH 114/381] compositor: fix illegal memory access in blend function with negative ypos https://bugzilla.gnome.org/show_bug.cgi?id=741115 --- gst/compositor/blend.c | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/gst/compositor/blend.c b/gst/compositor/blend.c index 7fe8e53fd8..721c766e68 100644 --- a/gst/compositor/blend.c +++ b/gst/compositor/blend.c @@ -86,9 +86,11 @@ method##_ ##name (GstVideoFrame * srcframe, gint xpos, gint ypos, \ src_height = dest_height - ypos; \ } \ \ - dest = dest + 4 * xpos + (ypos * dest_stride); \ + if (src_height > 0 && src_width > 0) { \ + dest = dest + 4 * xpos + (ypos * dest_stride); \ \ - LOOP (dest, src, src_height, src_width, src_stride, dest_stride, s_alpha); \ + LOOP (dest, src, src_height, src_width, src_stride, dest_stride, s_alpha); \ + } \ } #define BLEND_A32_LOOP(name, method) \ @@ -268,23 +270,23 @@ blend_##format_name (GstVideoFrame * srcframe, gint xpos, gint ypos, \ xpos = 0; \ } \ if (ypos < 0) { \ - yoffset += -ypos; \ + yoffset = -ypos; \ b_src_height -= -ypos; \ ypos = 0; \ } \ /* If x or y offset are larger then the source it's outside of the picture */ \ - if (xoffset > src_width || yoffset > src_height) { \ + if (xoffset >= src_width || yoffset >= src_height) { \ return; \ } \ \ /* adjust width/height if the src is bigger than dest */ \ - if (xpos + src_width > dest_width) { \ + if (xpos + b_src_width > dest_width) { \ b_src_width = dest_width - xpos; \ } \ - if (ypos + src_height > dest_height) { \ + if (ypos + b_src_height > dest_height) { \ b_src_height = dest_height - ypos; \ } \ - if (b_src_width < 0 || b_src_height < 0) { \ + if (b_src_width <= 0 || b_src_height <= 0) { \ return; \ } \ \ From 38a06d0a33a3f36bca1b765e73ee383e689687ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim-Philipp=20M=C3=BCller?= Date: Mon, 26 Jan 2015 09:22:23 +0000 Subject: [PATCH 115/381] videoaggregator: fix crash when receiving buffer without timestamps Unset out buffer in clip function when we unref the buffer to be clipped, otherwise aggregator will continue to use the already- freed buffer. Fixes crash when buffers without timestamps are being fed to aggregator. Partly because aggregator ignores the error flow return. https://bugzilla.gnome.org/show_bug.cgi?id=743334 --- gst-libs/gst/video/gstvideoaggregator.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index e850b6f7bb..005edcaa7b 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -1512,8 +1512,9 @@ gst_videoaggregator_sink_clip (GstAggregator * agg, start_time = GST_BUFFER_TIMESTAMP (buf); if (start_time == -1) { - GST_DEBUG_OBJECT (pad, "Timestamped buffers required!"); + GST_WARNING_OBJECT (pad, "Timestamped buffers required!"); gst_buffer_unref (buf); + *outbuf = NULL; return GST_FLOW_ERROR; } From 4527811b1f6ab981bc3c2681c062ee60fb2ec7e9 Mon Sep 17 00:00:00 2001 From: Nirbheek Chauhan Date: Thu, 29 Jan 2015 02:28:38 +0530 Subject: [PATCH 116/381] videoaggregator: If getting a timeout before having caps, just advance our position This can happen if this is a live pipeline and no source produced any buffer and sent no caps until an output buffer should've been produced according to the latency. This fix is similar in spirit to commit be7034d1 by Sebastian for audiomixer. --- gst-libs/gst/video/gstvideoaggregator.c | 42 +++++++++++++++++++++++-- 1 file changed, 40 insertions(+), 2 deletions(-) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index 005edcaa7b..871103f4da 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -760,6 +760,13 @@ gst_videoaggregator_update_src_caps (GstVideoAggregator * vagg) GST_VIDEO_AGGREGATOR_GET_CLASS (vagg)->negotiated_caps (vagg, caps); } gst_caps_unref (caps); + } else { + /* We couldn't decide the output video info because the sinkpads don't have + * all the caps yet, so we mark the pad as needing a reconfigure. This + * allows aggregate() to skip ahead a bit and try again later. */ + GST_DEBUG_OBJECT (vagg, "Couldn't decide output video info"); + gst_pad_mark_reconfigure (agg->srcpad); + ret = FALSE; } done: @@ -1261,8 +1268,39 @@ gst_videoaggregator_aggregate (GstAggregator * agg, gboolean timeout) ret = gst_videoaggregator_update_src_caps (vagg); if (!ret) { - GST_VIDEO_AGGREGATOR_UNLOCK (vagg); - return GST_FLOW_NOT_NEGOTIATED; + if (timeout && gst_pad_needs_reconfigure (agg->srcpad)) { + guint64 frame_duration; + gint fps_d, fps_n; + + GST_DEBUG_OBJECT (vagg, + "Got timeout before receiving any caps, don't output anything"); + + if (agg->segment.position == -1) { + if (agg->segment.rate > 0.0) + agg->segment.position = agg->segment.start; + else + agg->segment.position = agg->segment.stop; + } + + /* Advance position */ + fps_d = GST_VIDEO_INFO_FPS_D (&vagg->info) ? + GST_VIDEO_INFO_FPS_D (&vagg->info) : 1; + fps_n = GST_VIDEO_INFO_FPS_N (&vagg->info) ? + GST_VIDEO_INFO_FPS_N (&vagg->info) : 25; + /* Default to 25/1 if no "best fps" is known */ + frame_duration = gst_util_uint64_scale (GST_SECOND, fps_d, fps_n); + if (agg->segment.rate > 0.0) + agg->segment.position += frame_duration; + else if (agg->segment.position > frame_duration) + agg->segment.position -= frame_duration; + else + agg->segment.position = 0; + GST_VIDEO_AGGREGATOR_UNLOCK (vagg); + return GST_FLOW_OK; + } else { + GST_VIDEO_AGGREGATOR_UNLOCK (vagg); + return GST_FLOW_NOT_NEGOTIATED; + } } } From f5d63b031d496472d6aed8f5627d26b50d7035bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Olivier=20Cr=C3=AAte?= Date: Wed, 14 Jan 2015 14:45:06 -0500 Subject: [PATCH 117/381] videoaggregator: Lock access to members of GstAggregatorPad Take the pad's object lock before accessing members of the GstAggregatorPad structure. https://bugzilla.gnome.org/show_bug.cgi?id=742684 --- gst-libs/gst/video/gstvideoaggregator.c | 38 +++++++++++++++---------- 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index 871103f4da..b16f1c100b 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -956,15 +956,17 @@ gst_videoaggregator_fill_queues (GstVideoAggregator * vagg, GST_OBJECT_LOCK (vagg); for (l = GST_ELEMENT (vagg)->sinkpads; l; l = l->next) { GstVideoAggregatorPad *pad = l->data; - GstSegment *segment; + GstSegment segment; GstAggregatorPad *bpad; GstBuffer *buf; GstVideoInfo *vinfo; gboolean is_eos; bpad = GST_AGGREGATOR_PAD (pad); - segment = &bpad->segment; + GST_OBJECT_LOCK (bpad); + segment = bpad->segment; is_eos = bpad->eos; + GST_OBJECT_UNLOCK (bpad); if (!is_eos) eos = FALSE; buf = gst_aggregator_pad_get_buffer (bpad); @@ -1010,12 +1012,12 @@ gst_videoaggregator_fill_queues (GstVideoAggregator * vagg, end_time += start_time; /* convert from duration to position */ /* Check if it's inside the segment */ - if (start_time >= segment->stop || end_time < segment->start) { + if (start_time >= segment.stop || end_time < segment.start) { GST_DEBUG_OBJECT (pad, "Buffer outside the segment : segment: [%" GST_TIME_FORMAT " -- %" GST_TIME_FORMAT "]" " Buffer [%" GST_TIME_FORMAT " -- %" - GST_TIME_FORMAT "]", GST_TIME_ARGS (segment->stop), - GST_TIME_ARGS (segment->start), GST_TIME_ARGS (start_time), + GST_TIME_FORMAT "]", GST_TIME_ARGS (segment.stop), + GST_TIME_ARGS (segment.start), GST_TIME_ARGS (start_time), GST_TIME_ARGS (end_time)); gst_buffer_unref (buf); @@ -1027,13 +1029,13 @@ gst_videoaggregator_fill_queues (GstVideoAggregator * vagg, } /* Clip to segment and convert to running time */ - start_time = MAX (start_time, segment->start); - if (segment->stop != -1) - end_time = MIN (end_time, segment->stop); + start_time = MAX (start_time, segment.start); + if (segment.stop != -1) + end_time = MIN (end_time, segment.stop); start_time = - gst_segment_to_running_time (segment, GST_FORMAT_TIME, start_time); + gst_segment_to_running_time (&segment, GST_FORMAT_TIME, start_time); end_time = - gst_segment_to_running_time (segment, GST_FORMAT_TIME, end_time); + gst_segment_to_running_time (&segment, GST_FORMAT_TIME, end_time); g_assert (start_time != -1 && end_time != -1); /* Convert to the output segment rate */ @@ -1128,13 +1130,13 @@ prepare_frames (GstVideoAggregator * vagg, GstVideoAggregatorPad * pad) if (pad->buffer != NULL) { GstClockTime timestamp; gint64 stream_time; - GstSegment *seg; - - seg = &bpad->segment; timestamp = GST_BUFFER_TIMESTAMP (pad->buffer); - stream_time = gst_segment_to_stream_time (seg, GST_FORMAT_TIME, timestamp); + GST_OBJECT_LOCK (bpad); + stream_time = gst_segment_to_stream_time (&bpad->segment, GST_FORMAT_TIME, + timestamp); + GST_OBJECT_UNLOCK (bpad); /* sync object properties on stream time */ if (GST_CLOCK_TIME_IS_VALID (stream_time)) @@ -1566,6 +1568,8 @@ gst_videoaggregator_sink_clip (GstAggregator * agg, return GST_FLOW_OK; } + GST_OBJECT_LOCK (bpad); + start_time = MAX (start_time, bpad->segment.start); start_time = gst_segment_to_running_time (&bpad->segment, GST_FORMAT_TIME, start_time); @@ -1584,10 +1588,14 @@ gst_videoaggregator_sink_clip (GstAggregator * agg, if (bpad->buffer != NULL && end_time < pad->priv->end_time) { gst_buffer_unref (buf); *outbuf = NULL; - return GST_FLOW_OK; + goto done; } *outbuf = buf; + +done: + + GST_OBJECT_UNLOCK (bpad); return GST_FLOW_OK; } From 008de39d3f284429f84bc776fdff837a9a9bdf8e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Olivier=20Cr=C3=AAte?= Date: Wed, 21 Jan 2015 19:43:12 -0500 Subject: [PATCH 118/381] aggregator: Document locking for gst_aggregator_get_latency_unlocked() Renamed it to _unlocked() to make it clear. https://bugzilla.gnome.org/show_bug.cgi?id=742684 --- gst-libs/gst/video/gstvideoaggregator.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index b16f1c100b..9cd233c2d3 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -879,7 +879,8 @@ gst_videoaggregator_update_qos (GstVideoAggregator * vagg, gdouble proportion, GST_TIME_ARGS (ABS (diff)), GST_TIME_ARGS (timestamp)); GST_OBJECT_LOCK (vagg); - gst_aggregator_get_latency (GST_AGGREGATOR (vagg), &live, NULL, NULL); + gst_aggregator_get_latency_unlocked (GST_AGGREGATOR (vagg), &live, NULL, + NULL); vagg->priv->proportion = proportion; if (G_LIKELY (timestamp != GST_CLOCK_TIME_NONE)) { From 83e1895dbfa4f8e6d590f88b0d6d9bab0ac85dfa Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Mon, 26 Jan 2015 11:25:54 +0100 Subject: [PATCH 119/381] aggregator: Hide GstAggregatorPad buffer and EOS fileds And add a getter for the EOS. The user should always use the various getters to access those fields https://bugzilla.gnome.org/show_bug.cgi?id=742684 --- gst-libs/gst/video/gstvideoaggregator.c | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index 9cd233c2d3..3ec08b974e 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -966,8 +966,9 @@ gst_videoaggregator_fill_queues (GstVideoAggregator * vagg, bpad = GST_AGGREGATOR_PAD (pad); GST_OBJECT_LOCK (bpad); segment = bpad->segment; - is_eos = bpad->eos; GST_OBJECT_UNLOCK (bpad); + is_eos = gst_aggregator_pad_is_eos (bpad); + if (!is_eos) eos = FALSE; buf = gst_aggregator_pad_get_buffer (bpad); @@ -1550,6 +1551,7 @@ gst_videoaggregator_sink_clip (GstAggregator * agg, { GstVideoAggregatorPad *pad = GST_VIDEO_AGGREGATOR_PAD (bpad); GstClockTime start_time, end_time; + GstBuffer *pbuf; start_time = GST_BUFFER_TIMESTAMP (buf); if (start_time == -1) { @@ -1586,10 +1588,15 @@ gst_videoaggregator_sink_clip (GstAggregator * agg, end_time *= ABS (agg->segment.rate); } - if (bpad->buffer != NULL && end_time < pad->priv->end_time) { - gst_buffer_unref (buf); - *outbuf = NULL; - goto done; + pbuf = gst_aggregator_pad_get_buffer (bpad); + if (pbuf != NULL) { + gst_buffer_unref (pbuf); + + if (end_time < pad->priv->end_time) { + gst_buffer_unref (buf); + *outbuf = NULL; + goto done; + } } *outbuf = buf; From 490444637f98702fa49461d89e2dd019673cce52 Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Mon, 26 Jan 2015 11:32:47 +0100 Subject: [PATCH 120/381] aggregator: More fixes around locking when accessing protected private fields In some more places we were accessing GstAggregator->segment and GstAggregator->seqnum without holding the GST_OBJECT_LOCK https://bugzilla.gnome.org/show_bug.cgi?id=742684 --- gst-libs/gst/video/gstvideoaggregator.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index 3ec08b974e..2a8a48c4dc 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -1615,8 +1615,8 @@ gst_videoaggregator_flush (GstAggregator * agg) GstVideoAggregator *vagg = GST_VIDEO_AGGREGATOR (agg); GST_INFO_OBJECT (agg, "Flushing"); - abs_rate = ABS (agg->segment.rate); GST_OBJECT_LOCK (vagg); + abs_rate = ABS (agg->segment.rate); for (l = GST_ELEMENT (vagg)->sinkpads; l; l = l->next) { GstVideoAggregatorPad *p = l->data; From 4860e179fa624daba8d1d4a3db04bed0e3a646fa Mon Sep 17 00:00:00 2001 From: Alessandro Decina Date: Wed, 28 Jan 2015 00:13:46 +1100 Subject: [PATCH 121/381] gl: initial support for texture targets other than GL_TEXTURE_2D Make GstGLMemory hold the texture target (tex_target) the texture it represents (tex_id) is bound to. Modify gst_gl_memory_wrapped_texture and gst_gl_download_perform_with_data to take the texture target as an argument. This change is needed to support wrapping textures created outside libgstgl, which might be bound to a target other than GL_TEXTURE_2D. For example on OSX textures coming from VideoToolbox have target GL_TEXTURE_RECTANGLE. With this change we still keep (and sometimes imply) GL_TEXTURE_2D as the target of textures created with libgstgl. API: modify GstGLMemory API: modify gst_gl_memory_wrapped_texture API: gst_gl_download_perform_with_data --- ext/gl/gstglmixer.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/ext/gl/gstglmixer.c b/ext/gl/gstglmixer.c index 5fdcc3db93..a122f8f278 100644 --- a/ext/gl/gstglmixer.c +++ b/ext/gl/gstglmixer.c @@ -1127,7 +1127,7 @@ gst_gl_mixer_process_textures (GstGLMixer * mix, GstBuffer * outbuf) { guint i; GList *walk; - guint out_tex; + guint out_tex, out_tex_target; gboolean res = TRUE; guint array_index = 0; GstVideoFrame out_frame; @@ -1153,6 +1153,8 @@ gst_gl_mixer_process_textures (GstGLMixer * mix, GstBuffer * outbuf) if (!to_download) { out_tex = *(guint *) out_frame.data[0]; + out_tex_target = + ((GstGLMemory *) gst_buffer_peek_memory (outbuf, 0))->tex_target; } else { GST_INFO ("Output Buffer does not contain correct memory, " "attempting to wrap for download"); @@ -1162,6 +1164,7 @@ gst_gl_mixer_process_textures (GstGLMixer * mix, GstBuffer * outbuf) gst_gl_download_set_format (mix->download, &out_frame.info); out_tex = mix->out_tex_id; + out_tex_target = GL_TEXTURE_2D; } GST_OBJECT_LOCK (mix); @@ -1215,8 +1218,8 @@ gst_gl_mixer_process_textures (GstGLMixer * mix, GstBuffer * outbuf) g_mutex_unlock (&priv->gl_resource_lock); if (to_download) { - if (!gst_gl_download_perform_with_data (mix->download, out_tex, - out_frame.data)) { + if (!gst_gl_download_perform_with_data (mix->download, + out_tex, out_tex_target, out_frame.data)) { GST_ELEMENT_ERROR (mix, RESOURCE, NOT_FOUND, ("%s", "Failed to download video frame"), (NULL)); res = FALSE; From 47feb5e011ec7b221e468a033c68dc781f836dc4 Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Tue, 3 Feb 2015 00:11:06 +1100 Subject: [PATCH 122/381] glupload/download/convert: provide transform_caps functions Allows finer grain decisions about formats and features at each stage of the pipeline. Also provide propose_allocation for glupload besed on the supported methods. --- ext/gl/gstglmixer.c | 202 +++++++++++++++++++++----------------------- 1 file changed, 94 insertions(+), 108 deletions(-) diff --git a/ext/gl/gstglmixer.c b/ext/gl/gstglmixer.c index a122f8f278..57135a662e 100644 --- a/ext/gl/gstglmixer.c +++ b/ext/gl/gstglmixer.c @@ -117,6 +117,46 @@ gst_gl_mixer_pad_finalize (GObject * object) G_OBJECT_CLASS (gst_gl_mixer_pad_parent_class)->finalize (object); } +static void +_init_upload (GstGLMixer * mix, GstGLMixerPad * pad) +{ + GstVideoAggregatorPad *vaggpad = GST_VIDEO_AGGREGATOR_PAD (pad); + + if (!pad->upload) { + GstCaps *in_caps = gst_pad_get_current_caps (GST_PAD (pad)); + GstCaps *upload_caps = gst_caps_copy (in_caps); + GstCapsFeatures *gl_features = + gst_caps_features_from_string (GST_CAPS_FEATURE_MEMORY_GL_MEMORY); + + pad->upload = gst_gl_upload_new (mix->context); + + gst_caps_set_features (upload_caps, 0, + gst_caps_features_copy (gl_features)); + gst_gl_upload_set_caps (pad->upload, in_caps, upload_caps); + gst_caps_unref (in_caps); + + if (!pad->convert) { + GstVideoInfo gl_info; + GstCaps *gl_caps; + + gst_video_info_set_format (&gl_info, + GST_VIDEO_FORMAT_RGBA, + GST_VIDEO_INFO_WIDTH (&vaggpad->info), + GST_VIDEO_INFO_HEIGHT (&vaggpad->info)); + gl_caps = gst_video_info_to_caps (&gl_info); + gst_caps_set_features (gl_caps, 0, gst_caps_features_copy (gl_features)); + + pad->convert = gst_gl_color_convert_new (mix->context); + + gst_gl_color_convert_set_caps (pad->convert, upload_caps, gl_caps); + gst_caps_unref (gl_caps); + } + + gst_caps_unref (upload_caps); + gst_caps_features_free (gl_features); + } +} + static void gst_gl_mixer_pad_get_property (GObject * object, guint prop_id, GValue * value, GParamSpec * pspec) @@ -153,7 +193,7 @@ _negotiated_caps (GstVideoAggregator * vagg, GstCaps * caps) } static gboolean -gst_gl_mixer_propose_allocation (GstGLMixer * mix, +gst_gl_mixer_propose_allocation (GstGLMixer * mix, GstGLMixerPad * pad, GstQuery * decide_query, GstQuery * query) { GstGLMixerClass *mix_class = GST_GL_MIXER_GET_CLASS (mix); @@ -163,11 +203,6 @@ gst_gl_mixer_propose_allocation (GstGLMixer * mix, guint size = 0; gboolean need_pool; GError *error = NULL; - GstStructure *gl_context; - gchar *platform, *gl_apis; - gpointer handle; - GstAllocator *allocator = NULL; - GstAllocationParams params; gst_query_parse_allocation (query, &caps, &need_pool); @@ -229,32 +264,12 @@ gst_gl_mixer_propose_allocation (GstGLMixer * mix, } /* we also support various metadata */ - gst_query_add_allocation_meta (query, GST_VIDEO_META_API_TYPE, 0); if (mix->context->gl_vtable->FenceSync) gst_query_add_allocation_meta (query, GST_GL_SYNC_META_API_TYPE, 0); - gl_apis = gst_gl_api_to_string (gst_gl_context_get_gl_api (mix->context)); - platform = - gst_gl_platform_to_string (gst_gl_context_get_gl_platform (mix->context)); - handle = (gpointer) gst_gl_context_get_gl_context (mix->context); + _init_upload (mix, pad); - gl_context = - gst_structure_new ("GstVideoGLTextureUploadMeta", "gst.gl.GstGLContext", - GST_GL_TYPE_CONTEXT, mix->context, "gst.gl.context.handle", - G_TYPE_POINTER, handle, "gst.gl.context.type", G_TYPE_STRING, platform, - "gst.gl.context.apis", G_TYPE_STRING, gl_apis, NULL); - gst_query_add_allocation_meta (query, - GST_VIDEO_GL_TEXTURE_UPLOAD_META_API_TYPE, gl_context); - - g_free (gl_apis); - g_free (platform); - gst_structure_free (gl_context); - - gst_allocation_params_init (¶ms); - - allocator = gst_allocator_find (GST_GL_MEMORY_ALLOCATOR); - gst_query_add_allocation_param (query, allocator, ¶ms); - gst_object_unref (allocator); + gst_gl_upload_propose_allocation (pad->upload, decide_query, query); return TRUE; @@ -306,26 +321,9 @@ static GstCaps * gst_gl_mixer_set_caps_features (const GstCaps * caps, const gchar * feature_name) { - GstCaps *tmp = gst_caps_copy (caps); - guint n = gst_caps_get_size (tmp); - guint i = 0; - - for (i = 0; i < n; i++) { - GstCapsFeatures *features = gst_caps_get_features (tmp, i); - if (features) { - guint n_f = gst_caps_features_get_size (features); - guint j = 0; - for (j = 0; j < n_f; j++) { - gst_caps_features_remove_id (features, - gst_caps_features_get_nth_id (features, j)); - } - } - - gst_caps_features_add (features, feature_name); - gst_caps_set_simple (tmp, "format", G_TYPE_STRING, "RGBA", NULL); - } - - return tmp; + GstCaps *ret = gst_gl_caps_replace_all_caps_features (caps, feature_name); + gst_caps_set_simple (ret, "format", G_TYPE_STRING, "RGBA", NULL); + return ret; } /* copies the given caps */ @@ -367,30 +365,25 @@ gst_gl_mixer_caps_remove_format_info (GstCaps * caps) GstCaps * gst_gl_mixer_update_caps (GstGLMixer * mix, GstCaps * caps) { - GstCaps *result = NULL; - GstCaps *glcaps = gst_gl_mixer_set_caps_features (caps, - GST_CAPS_FEATURE_MEMORY_GL_MEMORY); -#if GST_GL_HAVE_PLATFORM_EGL - GstCaps *eglcaps = gst_gl_mixer_set_caps_features (caps, - GST_CAPS_FEATURE_MEMORY_EGL_IMAGE); -#endif - GstCaps *uploadcaps = gst_gl_mixer_set_caps_features (caps, - GST_CAPS_FEATURE_META_GST_VIDEO_GL_TEXTURE_UPLOAD_META); - GstCaps *raw_caps = - gst_caps_from_string (GST_VIDEO_CAPS_MAKE (GST_GL_COLOR_CONVERT_FORMATS)); + GstCaps *result, *tmp, *gl_caps; - result = gst_caps_new_empty (); + gl_caps = gst_caps_from_string ("video/x-raw(memory:GLMemory),format=RGBA"); - result = gst_caps_merge (result, glcaps); -#if GST_GL_HAVE_PLATFORM_EGL - result = gst_caps_merge (result, eglcaps); -#endif - result = gst_caps_merge (result, uploadcaps); - result = gst_caps_merge (result, raw_caps); + result = + gst_gl_color_convert_transform_caps (mix->context, GST_PAD_SRC, gl_caps, + NULL); + tmp = result; + GST_DEBUG_OBJECT (mix, "convert returned caps %" GST_PTR_FORMAT, tmp); - result = gst_caps_merge (result, gst_gl_mixer_caps_remove_format_info (caps)); + result = gst_gl_mixer_caps_remove_format_info (tmp); + gst_caps_unref (tmp); + tmp = result; + GST_DEBUG_OBJECT (mix, "remove format returned caps %" GST_PTR_FORMAT, tmp); - GST_DEBUG_OBJECT (mix, "returning %" GST_PTR_FORMAT, result); + result = gst_gl_upload_transform_caps (mix->context, GST_PAD_SRC, tmp, NULL); + gst_caps_unref (tmp); + tmp = result; + GST_DEBUG_OBJECT (mix, "transfer returned caps %" GST_PTR_FORMAT, tmp); return result; } @@ -398,7 +391,7 @@ gst_gl_mixer_update_caps (GstGLMixer * mix, GstCaps * caps) static GstCaps * gst_gl_mixer_pad_sink_getcaps (GstPad * pad, GstGLMixer * mix, GstCaps * filter) { - GstCaps *srccaps; + GstCaps *sinkcaps; GstCaps *template_caps; GstCaps *filtered_caps; GstCaps *returned_caps; @@ -406,17 +399,18 @@ gst_gl_mixer_pad_sink_getcaps (GstPad * pad, GstGLMixer * mix, GstCaps * filter) template_caps = gst_pad_get_pad_template_caps (pad); - srccaps = gst_pad_get_current_caps (pad); - if (srccaps == NULL) { + sinkcaps = gst_pad_get_current_caps (pad); + if (sinkcaps == NULL) { had_current_caps = FALSE; - srccaps = template_caps; + sinkcaps = template_caps; } else { - srccaps = gst_caps_merge (srccaps, gst_gl_mixer_update_caps (mix, srccaps)); + sinkcaps = + gst_caps_merge (sinkcaps, gst_gl_mixer_update_caps (mix, sinkcaps)); } - filtered_caps = srccaps; + filtered_caps = sinkcaps; if (filter) - filtered_caps = gst_caps_intersect (srccaps, filter); + filtered_caps = gst_caps_intersect (sinkcaps, filter); returned_caps = gst_caps_intersect (filtered_caps, template_caps); if (filter) @@ -436,6 +430,7 @@ gst_gl_mixer_sink_query (GstAggregator * agg, GstAggregatorPad * bpad, gboolean ret = FALSE; GstGLMixer *mix = GST_GL_MIXER (agg); GstGLMixerClass *mix_class = GST_GL_MIXER_GET_CLASS (mix); + GstGLMixerPad *pad = GST_GL_MIXER_PAD (bpad); GST_TRACE ("QUERY %" GST_PTR_FORMAT, query); @@ -481,7 +476,7 @@ gst_gl_mixer_sink_query (GstAggregator * agg, GstAggregatorPad * bpad, decide_query); /* pass the query to the propose_allocation vmethod if any */ - ret = gst_gl_mixer_propose_allocation (mix, decide_query, query); + ret = gst_gl_mixer_propose_allocation (mix, pad, decide_query, query); if (decide_query) gst_query_unref (decide_query); @@ -721,7 +716,8 @@ gst_gl_mixer_src_activate_mode (GstAggregator * aggregator, GstPadMode mode, static gboolean gst_gl_mixer_query_caps (GstPad * pad, GstAggregator * agg, GstQuery * query) { - GstCaps *filter, *current_caps, *retcaps; + GstGLMixer *mix = GST_GL_MIXER (agg); + GstCaps *filter, *current_caps, *retcaps, *gl_caps; gst_query_parse_caps (query, &filter); @@ -729,12 +725,30 @@ gst_gl_mixer_query_caps (GstPad * pad, GstAggregator * agg, GstQuery * query) if (current_caps == NULL) current_caps = gst_pad_get_pad_template_caps (agg->srcpad); + /* convert from current caps to GLMemory caps */ + gl_caps = + gst_caps_merge (gst_caps_merge (gst_gl_mixer_set_caps_features + (current_caps, GST_CAPS_FEATURE_MEMORY_GL_MEMORY), + gst_gl_mixer_set_caps_features (current_caps, + GST_CAPS_FEATURE_MEMORY_EGL_IMAGE)), + gst_gl_mixer_set_caps_features (current_caps, + GST_CAPS_FEATURE_META_GST_VIDEO_GL_TEXTURE_UPLOAD_META)); + retcaps = + gst_gl_download_transform_caps (mix->context, GST_PAD_SINK, current_caps, + NULL); + retcaps = gst_caps_merge (gl_caps, retcaps); + gst_caps_unref (current_caps); + current_caps = retcaps; + retcaps = gst_gl_mixer_caps_remove_format_info (current_caps); gst_caps_unref (current_caps); - if (filter) - retcaps = + if (filter) { + current_caps = gst_caps_intersect_full (filter, retcaps, GST_CAPS_INTERSECT_FIRST); + gst_caps_unref (retcaps); + retcaps = current_caps; + } gst_query_set_caps_result (query, retcaps); gst_caps_unref (retcaps); @@ -1054,8 +1068,6 @@ _default_pad_upload_buffer (GstGLMixer * mix, GstGLMixerFrameData * frame, GstVideoAggregatorPad *vaggpad = GST_VIDEO_AGGREGATOR_PAD (frame->pad); GstGLMixerPad *pad = frame->pad; GstBuffer *uploaded_buf, *gl_buffer; - GstCaps *gl_caps; - GstCapsFeatures *gl_features; GstVideoInfo gl_info; GstVideoFrame gl_frame; GstGLSyncMeta *sync_meta; @@ -1064,34 +1076,8 @@ _default_pad_upload_buffer (GstGLMixer * mix, GstGLMixerFrameData * frame, GST_VIDEO_FORMAT_RGBA, GST_VIDEO_INFO_WIDTH (&vaggpad->info), GST_VIDEO_INFO_HEIGHT (&vaggpad->info)); - gl_features = - gst_caps_features_from_string (GST_CAPS_FEATURE_MEMORY_GL_MEMORY); - gl_caps = gst_video_info_to_caps (&gl_info); - gst_caps_set_features (gl_caps, 0, gst_caps_features_copy (gl_features)); - - if (!pad->upload) { - GstCaps *in_caps = gst_pad_get_current_caps (GST_PAD (pad)); - GstCaps *upload_caps = gst_caps_copy (in_caps); - - pad->upload = gst_gl_upload_new (mix->context); - - gst_caps_set_features (upload_caps, 0, - gst_caps_features_copy (gl_features)); - gst_gl_upload_set_caps (pad->upload, in_caps, upload_caps); - - if (!pad->convert) { - pad->convert = gst_gl_color_convert_new (mix->context); - - gst_gl_color_convert_set_caps (pad->convert, upload_caps, gl_caps); - } - - gst_caps_unref (upload_caps); - gst_caps_unref (in_caps); - } - - gst_caps_features_free (gl_features); - gst_caps_unref (gl_caps); + _init_upload (mix, pad); sync_meta = gst_buffer_get_gl_sync_meta (vaggpad->buffer); if (sync_meta) From bae0d5755af07bc160c7bf391d4eab4fcc65193d Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Tue, 3 Feb 2015 15:30:08 +1100 Subject: [PATCH 123/381] gl: remove the egl caps from the src pads we don't actually support producing EGLImage buffers anywhere. --- ext/gl/gstglmixer.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/ext/gl/gstglmixer.c b/ext/gl/gstglmixer.c index 57135a662e..2cdf46fb92 100644 --- a/ext/gl/gstglmixer.c +++ b/ext/gl/gstglmixer.c @@ -537,6 +537,10 @@ static GstStaticPadTemplate sink_factory = GST_STATIC_PAD_TEMPLATE ("sink_%u", GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES (GST_CAPS_FEATURE_MEMORY_GL_MEMORY, "RGBA") "; " +#if GST_GL_HAVE_PLATFORM_EGL + GST_VIDEO_CAPS_MAKE_WITH_FEATURES (GST_CAPS_FEATURE_MEMORY_EGL_IMAGE, + "RGBA") "; " +#endif GST_VIDEO_CAPS_MAKE_WITH_FEATURES (GST_CAPS_FEATURE_META_GST_VIDEO_GL_TEXTURE_UPLOAD_META, "RGBA") @@ -727,10 +731,8 @@ gst_gl_mixer_query_caps (GstPad * pad, GstAggregator * agg, GstQuery * query) /* convert from current caps to GLMemory caps */ gl_caps = - gst_caps_merge (gst_caps_merge (gst_gl_mixer_set_caps_features - (current_caps, GST_CAPS_FEATURE_MEMORY_GL_MEMORY), - gst_gl_mixer_set_caps_features (current_caps, - GST_CAPS_FEATURE_MEMORY_EGL_IMAGE)), + gst_caps_merge (gst_gl_mixer_set_caps_features + (current_caps, GST_CAPS_FEATURE_MEMORY_GL_MEMORY), gst_gl_mixer_set_caps_features (current_caps, GST_CAPS_FEATURE_META_GST_VIDEO_GL_TEXTURE_UPLOAD_META)); retcaps = From 302353307acd37677ba497202dc03d58bd12035e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim-Philipp=20M=C3=BCller?= Date: Thu, 5 Feb 2015 15:48:41 +0000 Subject: [PATCH 124/381] tests: compositor: add unit test for proper segment.base handling As adjusted by gst_pad_set_offset(), or when doing segment seeks or looping for example. --- tests/check/elements/compositor.c | 65 +++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/tests/check/elements/compositor.c b/tests/check/elements/compositor.c index 9ef52baeb0..0e7380fe77 100644 --- a/tests/check/elements/compositor.c +++ b/tests/check/elements/compositor.c @@ -1035,6 +1035,68 @@ GST_START_TEST (test_flush_start_flush_stop) GST_END_TEST; +#ifndef GST_DISABLE_PARSE +GST_START_TEST (test_segment_base_handling) +{ + GstElement *pipeline, *sink, *mix, *src1, *src2; + GstPad *srcpad, *sinkpad; + GstClockTime end_time; + GstSample *last_sample = NULL; + GstSample *sample; + GstBuffer *buf; + GstCaps *caps; + + caps = gst_caps_new_simple ("video/x-raw", "width", G_TYPE_INT, 16, + "height", G_TYPE_INT, 16, "framerate", GST_TYPE_FRACTION, 30, 1, NULL); + + /* each source generates 5 seconds of data, src2 shifted by 5 seconds */ + pipeline = gst_pipeline_new ("pipeline"); + mix = gst_element_factory_make ("compositor", "compositor"); + sink = gst_element_factory_make ("appsink", "sink"); + g_object_set (sink, "caps", caps, "sync", FALSE, NULL); + gst_caps_unref (caps); + src1 = gst_element_factory_make ("videotestsrc", "src1"); + g_object_set (src1, "num-buffers", 30 * 5, "pattern", 2, NULL); + src2 = gst_element_factory_make ("videotestsrc", "src2"); + g_object_set (src2, "num-buffers", 30 * 5, "pattern", 2, NULL); + gst_bin_add_many (GST_BIN (pipeline), src1, src2, mix, sink, NULL); + fail_unless (gst_element_link (mix, sink)); + + srcpad = gst_element_get_static_pad (src1, "src"); + sinkpad = gst_element_get_request_pad (mix, "sink_1"); + fail_unless (gst_pad_link (srcpad, sinkpad) == GST_PAD_LINK_OK); + gst_object_unref (sinkpad); + gst_object_unref (srcpad); + + srcpad = gst_element_get_static_pad (src2, "src"); + sinkpad = gst_element_get_request_pad (mix, "sink_2"); + fail_unless (gst_pad_link (srcpad, sinkpad) == GST_PAD_LINK_OK); + gst_pad_set_offset (sinkpad, 5 * GST_SECOND); + gst_object_unref (sinkpad); + gst_object_unref (srcpad); + + gst_element_set_state (pipeline, GST_STATE_PLAYING); + + do { + g_signal_emit_by_name (sink, "pull-sample", &sample); + if (sample == NULL) + break; + if (last_sample) + gst_sample_unref (last_sample); + last_sample = sample; + } while (TRUE); + + buf = gst_sample_get_buffer (last_sample); + end_time = GST_BUFFER_TIMESTAMP (buf) + GST_BUFFER_DURATION (buf); + fail_unless_equals_int64 (end_time, 10 * GST_SECOND); + gst_sample_unref (last_sample); + + gst_element_set_state (pipeline, GST_STATE_NULL); + gst_object_unref (pipeline); +} + +GST_END_TEST; +#endif static Suite * compositor_suite (void) @@ -1054,6 +1116,9 @@ compositor_suite (void) tcase_add_test (tc_chain, test_duration_unknown_overrides); tcase_add_test (tc_chain, test_loop); tcase_add_test (tc_chain, test_flush_start_flush_stop); +#ifndef GST_DISABLE_PARSE + tcase_add_test (tc_chain, test_segment_base_handling); +#endif /* Use a longer timeout */ #ifdef HAVE_VALGRIND From bde4292e90aa19481b660254fe919c8ee53d134e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim-Philipp=20M=C3=BCller?= Date: Sun, 8 Feb 2015 14:32:22 +0000 Subject: [PATCH 125/381] videoaggregator: don't set element factory details This is an abstract base class, not an element. --- gst-libs/gst/video/gstvideoaggregator.c | 8 -------- 1 file changed, 8 deletions(-) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index 2a8a48c4dc..db413ca3f9 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -1947,14 +1947,6 @@ gst_videoaggregator_class_init (GstVideoAggregatorClass * klass) gstelement_class->release_pad = GST_DEBUG_FUNCPTR (gst_videoaggregator_release_pad); - gst_element_class_set_static_metadata (gstelement_class, - "Video aggregator base class", "Filter/Editor/Video", - "Aggregate multiple video streams", - "Wim Taymans , " - "Sebastian Dröge , " - "Mathieu Duponchelle , " - "Thibault Saunier "); - agg_class->sinkpads_type = GST_TYPE_VIDEO_AGGREGATOR_PAD; agg_class->start = gst_videoaggregator_start; agg_class->stop = gst_videoaggregator_stop; From b4ee792fc2f9911c5ae85565deab03af490e38af Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Mon, 9 Feb 2015 00:59:30 +1100 Subject: [PATCH 126/381] videoaggregator: fix buffer selection when duration=-1 If the src framerate and videoaggreator's output framerate were different, then we were taking every single buffer that had duration=-1 as it came in regardless of the buffer's start time. This caused the src to possibly run at a different speed to the output frames. https://bugzilla.gnome.org/show_bug.cgi?id=744096 --- gst-libs/gst/video/gstvideoaggregator.c | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index db413ca3f9..bd6eb32c50 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -1000,6 +1000,25 @@ gst_videoaggregator_fill_queues (GstVideoAggregator * vagg, end_time = GST_BUFFER_DURATION (buf); if (end_time == -1) { + start_time = MAX (start_time, segment.start); + start_time = + gst_segment_to_running_time (&segment, GST_FORMAT_TIME, start_time); + + if (start_time >= output_end_time) { + GST_DEBUG_OBJECT (pad, "buffer duration is -1, start_time >= " + "output_end_time. Keeping previous buffer"); + gst_buffer_unref (buf); + continue; + } else if (start_time < output_start_time) { + GST_DEBUG_OBJECT (pad, "buffer duration is -1, start_time < " + "output_start_time. Discarding old buffer"); + gst_buffer_replace (&pad->buffer, buf); + gst_buffer_unref (buf); + buf = gst_aggregator_pad_steal_buffer (bpad); + gst_buffer_unref (buf); + need_more_data = TRUE; + continue; + } gst_buffer_unref (buf); buf = gst_aggregator_pad_steal_buffer (bpad); gst_buffer_replace (&pad->buffer, buf); From 0d89c672e81973d7e3c887b987fc09a5c35508dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Tue, 10 Feb 2015 10:56:37 +0100 Subject: [PATCH 127/381] videoaggregator: When receiving timeout before caps, make sure to also advance our frame counter Otherwise we will directly go EOS on the next non-timeout. --- gst-libs/gst/video/gstvideoaggregator.c | 1 + 1 file changed, 1 insertion(+) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index bd6eb32c50..7f1381610a 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -1318,6 +1318,7 @@ gst_videoaggregator_aggregate (GstAggregator * agg, gboolean timeout) agg->segment.position -= frame_duration; else agg->segment.position = 0; + vagg->priv->nframes++; GST_VIDEO_AGGREGATOR_UNLOCK (vagg); return GST_FLOW_OK; } else { From d9a267f03054f3c6b128cca79acfa7f7005eb5e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Wed, 11 Feb 2015 14:16:21 +0100 Subject: [PATCH 128/381] Improve and fix LATENCY query handling This now follows the design docs everywhere, especially the maximum latency handling. https://bugzilla.gnome.org/show_bug.cgi?id=744106 --- gst-libs/gst/video/gstvideoaggregator.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index 7f1381610a..d9235c3e4a 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -598,13 +598,15 @@ gst_videoaggregator_src_setcaps (GstVideoAggregator * vagg, GstCaps * caps) if (vagg->priv->current_caps == NULL || gst_caps_is_equal (caps, vagg->priv->current_caps) == FALSE) { + GstClockTime latency; + gst_caps_replace (&vagg->priv->current_caps, caps); GST_VIDEO_AGGREGATOR_UNLOCK (vagg); gst_aggregator_set_src_caps (agg, caps); - gst_aggregator_set_latency (agg, gst_util_uint64_scale (GST_SECOND, - GST_VIDEO_INFO_FPS_D (&info), GST_VIDEO_INFO_FPS_N (&info)), - GST_CLOCK_TIME_NONE); + latency = gst_util_uint64_scale (GST_SECOND, + GST_VIDEO_INFO_FPS_D (&info), GST_VIDEO_INFO_FPS_N (&info)); + gst_aggregator_set_latency (agg, latency, latency); GST_VIDEO_AGGREGATOR_LOCK (vagg); } From 0d510b2da90c02e60188754c7ac9274d4d18aa57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim-Philipp=20M=C3=BCller?= Date: Thu, 12 Feb 2015 15:48:07 +0000 Subject: [PATCH 129/381] tests: remove GST_DISABLE_PARSE guards from two tests that don't require it --- tests/check/elements/compositor.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/check/elements/compositor.c b/tests/check/elements/compositor.c index 0e7380fe77..dab259bdf2 100644 --- a/tests/check/elements/compositor.c +++ b/tests/check/elements/compositor.c @@ -1035,7 +1035,6 @@ GST_START_TEST (test_flush_start_flush_stop) GST_END_TEST; -#ifndef GST_DISABLE_PARSE GST_START_TEST (test_segment_base_handling) { GstElement *pipeline, *sink, *mix, *src1, *src2; @@ -1096,7 +1095,6 @@ GST_START_TEST (test_segment_base_handling) } GST_END_TEST; -#endif static Suite * compositor_suite (void) @@ -1116,9 +1114,7 @@ compositor_suite (void) tcase_add_test (tc_chain, test_duration_unknown_overrides); tcase_add_test (tc_chain, test_loop); tcase_add_test (tc_chain, test_flush_start_flush_stop); -#ifndef GST_DISABLE_PARSE tcase_add_test (tc_chain, test_segment_base_handling); -#endif /* Use a longer timeout */ #ifdef HAVE_VALGRIND From 8b87f3f2fe26f2d6fe573f01c0821070f8be570c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim-Philipp=20M=C3=BCller?= Date: Fri, 13 Feb 2015 16:03:53 +0000 Subject: [PATCH 130/381] videoaggregator: use new gst_aggregator_pad_drop_buffer() --- gst-libs/gst/video/gstvideoaggregator.c | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index d9235c3e4a..71458ed9d5 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -993,8 +993,7 @@ gst_videoaggregator_fill_queues (GstVideoAggregator * vagg, || (pad->buffer && start_time < GST_BUFFER_TIMESTAMP (pad->buffer))) { GST_DEBUG_OBJECT (pad, "Buffer from the past, dropping"); gst_buffer_unref (buf); - buf = gst_aggregator_pad_steal_buffer (bpad); - gst_buffer_unref (buf); + gst_aggregator_pad_drop_buffer (bpad); need_more_data = TRUE; continue; } @@ -1016,8 +1015,7 @@ gst_videoaggregator_fill_queues (GstVideoAggregator * vagg, "output_start_time. Discarding old buffer"); gst_buffer_replace (&pad->buffer, buf); gst_buffer_unref (buf); - buf = gst_aggregator_pad_steal_buffer (bpad); - gst_buffer_unref (buf); + gst_aggregator_pad_drop_buffer (bpad); need_more_data = TRUE; continue; } @@ -1044,8 +1042,7 @@ gst_videoaggregator_fill_queues (GstVideoAggregator * vagg, GST_TIME_ARGS (end_time)); gst_buffer_unref (buf); - buf = gst_aggregator_pad_steal_buffer (bpad); - gst_buffer_unref (buf); + gst_aggregator_pad_drop_buffer (bpad); need_more_data = TRUE; continue; @@ -1076,8 +1073,7 @@ gst_videoaggregator_fill_queues (GstVideoAggregator * vagg, if (pad->priv->end_time != -1 && pad->priv->end_time > end_time) { GST_DEBUG_OBJECT (pad, "Buffer from the past, dropping"); gst_buffer_unref (buf); - buf = gst_aggregator_pad_steal_buffer (bpad); - gst_buffer_unref (buf); + gst_aggregator_pad_drop_buffer (bpad); continue; } @@ -1091,9 +1087,7 @@ gst_videoaggregator_fill_queues (GstVideoAggregator * vagg, pad->priv->end_time = end_time; gst_buffer_unref (buf); - buf = gst_aggregator_pad_steal_buffer (bpad); - if (buf) - gst_buffer_unref (buf); + gst_aggregator_pad_drop_buffer (bpad); eos = FALSE; } else if (start_time >= output_end_time) { GST_DEBUG_OBJECT (pad, "Keeping buffer until %" GST_TIME_FORMAT, @@ -1110,9 +1104,7 @@ gst_videoaggregator_fill_queues (GstVideoAggregator * vagg, " out end %" GST_TIME_FORMAT, GST_TIME_ARGS (start_time), GST_TIME_ARGS (output_end_time)); gst_buffer_unref (buf); - buf = gst_aggregator_pad_steal_buffer (bpad); - if (buf) - gst_buffer_unref (buf); + gst_aggregator_pad_drop_buffer (bpad); need_more_data = TRUE; continue; From 3555c7bcd9c8ca232124eceaf26d5ee43225c4c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Fri, 6 Mar 2015 14:36:26 +0100 Subject: [PATCH 131/381] videoaggregator: Create new caps from the video-info In case the original caps were missing some optional fields like interlace-mode. We assume default values for those everywhere, but they can still cause negotiation to fail if a downstream element expects the field to be there and at a specific value. --- gst-libs/gst/video/gstvideoaggregator.c | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index 71458ed9d5..cf33ac3fe6 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -599,14 +599,24 @@ gst_videoaggregator_src_setcaps (GstVideoAggregator * vagg, GstCaps * caps) if (vagg->priv->current_caps == NULL || gst_caps_is_equal (caps, vagg->priv->current_caps) == FALSE) { GstClockTime latency; + GstCaps *full_caps; - gst_caps_replace (&vagg->priv->current_caps, caps); + /* Create new caps from the video-info, in case the original caps + * were missing some optional fields like interlace-mode. We assume + * default values for those everywhere, but they can still cause + * negotiation to fail if a downstream element expects the field to be + * there and at a specific value. + */ + full_caps = gst_video_info_to_caps (&info); + + gst_caps_replace (&vagg->priv->current_caps, full_caps); GST_VIDEO_AGGREGATOR_UNLOCK (vagg); - gst_aggregator_set_src_caps (agg, caps); + gst_aggregator_set_src_caps (agg, full_caps); latency = gst_util_uint64_scale (GST_SECOND, GST_VIDEO_INFO_FPS_D (&info), GST_VIDEO_INFO_FPS_N (&info)); gst_aggregator_set_latency (agg, latency, latency); + gst_caps_unref (full_caps); GST_VIDEO_AGGREGATOR_LOCK (vagg); } From 5dfa3c80091ef76c39bd041dd6e65078b5c1d2d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Fri, 6 Mar 2015 18:49:16 +0100 Subject: [PATCH 132/381] Revert "videoaggregator: Create new caps from the video-info" This reverts commit 78215be0dfbb4e8ed4f249e161a94c644328d28d. because it broke glvideomixer with custom caps features. --- gst-libs/gst/video/gstvideoaggregator.c | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index cf33ac3fe6..71458ed9d5 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -599,24 +599,14 @@ gst_videoaggregator_src_setcaps (GstVideoAggregator * vagg, GstCaps * caps) if (vagg->priv->current_caps == NULL || gst_caps_is_equal (caps, vagg->priv->current_caps) == FALSE) { GstClockTime latency; - GstCaps *full_caps; - /* Create new caps from the video-info, in case the original caps - * were missing some optional fields like interlace-mode. We assume - * default values for those everywhere, but they can still cause - * negotiation to fail if a downstream element expects the field to be - * there and at a specific value. - */ - full_caps = gst_video_info_to_caps (&info); - - gst_caps_replace (&vagg->priv->current_caps, full_caps); + gst_caps_replace (&vagg->priv->current_caps, caps); GST_VIDEO_AGGREGATOR_UNLOCK (vagg); - gst_aggregator_set_src_caps (agg, full_caps); + gst_aggregator_set_src_caps (agg, caps); latency = gst_util_uint64_scale (GST_SECOND, GST_VIDEO_INFO_FPS_D (&info), GST_VIDEO_INFO_FPS_N (&info)); gst_aggregator_set_latency (agg, latency, latency); - gst_caps_unref (full_caps); GST_VIDEO_AGGREGATOR_LOCK (vagg); } From f153b9ee590c9dfd494e16fbabcab28e8f42caf9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Fri, 6 Mar 2015 18:15:11 +0100 Subject: [PATCH 133/381] gl: Remove format info in glcolorconvert's transform_caps function instead of on every caller ... and let glmixer actually transform the caps it is supposed to transform instead of inventing new caps. --- ext/gl/gstglmixer.c | 51 ++------------------------------------------- 1 file changed, 2 insertions(+), 49 deletions(-) diff --git a/ext/gl/gstglmixer.c b/ext/gl/gstglmixer.c index 2cdf46fb92..36952f8701 100644 --- a/ext/gl/gstglmixer.c +++ b/ext/gl/gstglmixer.c @@ -326,60 +326,17 @@ gst_gl_mixer_set_caps_features (const GstCaps * caps, return ret; } -/* copies the given caps */ -static GstCaps * -gst_gl_mixer_caps_remove_format_info (GstCaps * caps) -{ - GstStructure *st; - GstCapsFeatures *f; - gint i, n; - GstCaps *res; - - res = gst_caps_new_empty (); - - n = gst_caps_get_size (caps); - for (i = 0; i < n; i++) { - st = gst_caps_get_structure (caps, i); - f = gst_caps_get_features (caps, i); - - /* If this is already expressed by the existing caps - * skip this structure */ - if (i > 0 && gst_caps_is_subset_structure_full (res, st, f)) - continue; - - st = gst_structure_copy (st); - /* Only remove format info for the cases when we can actually convert */ - if (!gst_caps_features_is_any (f) - && gst_caps_features_is_equal (f, - GST_CAPS_FEATURES_MEMORY_SYSTEM_MEMORY)) - gst_structure_remove_fields (st, "format", "colorimetry", "chroma-site", - NULL); - gst_structure_remove_fields (st, "width", "height", NULL); - - gst_caps_append_structure_full (res, st, gst_caps_features_copy (f)); - } - - return res; -} - GstCaps * gst_gl_mixer_update_caps (GstGLMixer * mix, GstCaps * caps) { - GstCaps *result, *tmp, *gl_caps; - - gl_caps = gst_caps_from_string ("video/x-raw(memory:GLMemory),format=RGBA"); + GstCaps *result, *tmp; result = - gst_gl_color_convert_transform_caps (mix->context, GST_PAD_SRC, gl_caps, + gst_gl_color_convert_transform_caps (mix->context, GST_PAD_SRC, caps, NULL); tmp = result; GST_DEBUG_OBJECT (mix, "convert returned caps %" GST_PTR_FORMAT, tmp); - result = gst_gl_mixer_caps_remove_format_info (tmp); - gst_caps_unref (tmp); - tmp = result; - GST_DEBUG_OBJECT (mix, "remove format returned caps %" GST_PTR_FORMAT, tmp); - result = gst_gl_upload_transform_caps (mix->context, GST_PAD_SRC, tmp, NULL); gst_caps_unref (tmp); tmp = result; @@ -740,10 +697,6 @@ gst_gl_mixer_query_caps (GstPad * pad, GstAggregator * agg, GstQuery * query) NULL); retcaps = gst_caps_merge (gl_caps, retcaps); gst_caps_unref (current_caps); - current_caps = retcaps; - - retcaps = gst_gl_mixer_caps_remove_format_info (current_caps); - gst_caps_unref (current_caps); if (filter) { current_caps = From f433695b5d9603d204fba70da6356dd7af2a2233 Mon Sep 17 00:00:00 2001 From: Nicolas Dufresne Date: Fri, 6 Mar 2015 20:11:00 -0500 Subject: [PATCH 134/381] glmixer: Don't share our downstream pool with upstream Pool cannot have multiple owner. This can lead to spurious pool was flushing error. https://bugzilla.gnome.org/show_bug.cgi?id=74570 --- ext/gl/gstglmixer.c | 25 +++---------------------- 1 file changed, 3 insertions(+), 22 deletions(-) diff --git a/ext/gl/gstglmixer.c b/ext/gl/gstglmixer.c index 36952f8701..82a685f765 100644 --- a/ext/gl/gstglmixer.c +++ b/ext/gl/gstglmixer.c @@ -197,7 +197,7 @@ gst_gl_mixer_propose_allocation (GstGLMixer * mix, GstGLMixerPad * pad, GstQuery * decide_query, GstQuery * query) { GstGLMixerClass *mix_class = GST_GL_MIXER_GET_CLASS (mix); - GstBufferPool *pool; + GstBufferPool *pool = NULL; GstStructure *config; GstCaps *caps; guint size = 0; @@ -209,26 +209,7 @@ gst_gl_mixer_propose_allocation (GstGLMixer * mix, GstGLMixerPad * pad, if (caps == NULL) goto no_caps; - if ((pool = mix->priv->pool)) - gst_object_ref (pool); - - if (pool != NULL) { - GstCaps *pcaps; - - /* we had a pool, check caps */ - GST_DEBUG_OBJECT (mix, "check existing pool caps"); - config = gst_buffer_pool_get_config (pool); - gst_buffer_pool_config_get_params (config, &pcaps, &size, NULL, NULL); - - if (!gst_caps_is_equal (caps, pcaps)) { - GST_DEBUG_OBJECT (mix, "pool has different caps"); - /* different caps, we can't use this pool */ - gst_object_unref (pool); - pool = NULL; - } - gst_structure_free (config); - } - + /* FIXME this is not thread safe, this method can be called in parallel */ if (!gst_gl_ensure_element_data (mix, &mix->display, &mix->other_context)) return FALSE; @@ -240,7 +221,7 @@ gst_gl_mixer_propose_allocation (GstGLMixer * mix, GstGLMixerPad * pad, goto context_error; } - if (pool == NULL && need_pool) { + if (need_pool) { GstVideoInfo info; if (!gst_video_info_from_caps (&info, caps)) From e387604d667419efa09bede276f67009aefcfab0 Mon Sep 17 00:00:00 2001 From: Nirbheek Chauhan Date: Tue, 20 Jan 2015 19:53:09 +0530 Subject: [PATCH 135/381] videoaggregator: While aggregating, sync values for all pads before converting frames --- gst-libs/gst/video/gstvideoaggregator.c | 54 ++++++++++++++----------- 1 file changed, 31 insertions(+), 23 deletions(-) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index 71458ed9d5..24ab93f47e 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -1136,33 +1136,38 @@ gst_videoaggregator_fill_queues (GstVideoAggregator * vagg, } static gboolean -prepare_frames (GstVideoAggregator * vagg, GstVideoAggregatorPad * pad) +sync_pad_values (GstVideoAggregator * vagg, GstVideoAggregatorPad * pad) { GstAggregatorPad *bpad = GST_AGGREGATOR_PAD (pad); + GstClockTime timestamp; + gint64 stream_time; + + if (pad->buffer == NULL) + return TRUE; + + timestamp = GST_BUFFER_TIMESTAMP (pad->buffer); + GST_OBJECT_LOCK (bpad); + stream_time = gst_segment_to_stream_time (&bpad->segment, GST_FORMAT_TIME, + timestamp); + GST_OBJECT_UNLOCK (bpad); + + /* sync object properties on stream time */ + if (GST_CLOCK_TIME_IS_VALID (stream_time)) + gst_object_sync_values (GST_OBJECT (pad), stream_time); + + return TRUE; +} + +static gboolean +prepare_frames (GstVideoAggregator * vagg, GstVideoAggregatorPad * pad) +{ GstVideoAggregatorPadClass *vaggpad_class = GST_VIDEO_AGGREGATOR_PAD_GET_CLASS (pad); - if (pad->buffer != NULL) { - GstClockTime timestamp; - gint64 stream_time; + if (pad->buffer == NULL || !vaggpad_class->prepare_frame) + return TRUE; - timestamp = GST_BUFFER_TIMESTAMP (pad->buffer); - - GST_OBJECT_LOCK (bpad); - stream_time = gst_segment_to_stream_time (&bpad->segment, GST_FORMAT_TIME, - timestamp); - GST_OBJECT_UNLOCK (bpad); - - /* sync object properties on stream time */ - if (GST_CLOCK_TIME_IS_VALID (stream_time)) - gst_object_sync_values (GST_OBJECT (pad), stream_time); - - if (vaggpad_class->prepare_frame) { - return vaggpad_class->prepare_frame (pad, vagg); - } - } - - return TRUE; + return vaggpad_class->prepare_frame (pad, vagg); } static gboolean @@ -1199,8 +1204,11 @@ gst_videoaggregator_do_aggregate (GstVideoAggregator * vagg, GST_BUFFER_TIMESTAMP (*outbuf) = output_start_time; GST_BUFFER_DURATION (*outbuf) = output_end_time - output_start_time; - /* Here we convert all the frames the subclass will have to aggregate - * and also sync pad properties to the stream time */ + /* Sync pad properties to the stream time */ + gst_aggregator_iterate_sinkpads (GST_AGGREGATOR (vagg), + (GstAggregatorPadForeachFunc) sync_pad_values, NULL); + + /* Convert all the frames the subclass has before aggregating */ gst_aggregator_iterate_sinkpads (GST_AGGREGATOR (vagg), (GstAggregatorPadForeachFunc) prepare_frames, NULL); From d175b0be59895befed841a15a0725a4619248d29 Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Wed, 11 Feb 2015 14:48:45 +1100 Subject: [PATCH 136/381] gl: add a new glbasemixer class below glmixer It deals with propagating the gl display/contexts throughout the application/pipeline --- ext/gl/gstglbasemixer.c | 734 +++++++++++++++++++++++++++++++++++++++ ext/gl/gstglbasemixer.h | 103 ++++++ ext/gl/gstglmixer.c | 472 ++++--------------------- ext/gl/gstglmixer.h | 58 +++- ext/gl/gstglvideomixer.h | 1 - 5 files changed, 955 insertions(+), 413 deletions(-) create mode 100644 ext/gl/gstglbasemixer.c create mode 100644 ext/gl/gstglbasemixer.h diff --git a/ext/gl/gstglbasemixer.c b/ext/gl/gstglbasemixer.c new file mode 100644 index 0000000000..39d6738d14 --- /dev/null +++ b/ext/gl/gstglbasemixer.c @@ -0,0 +1,734 @@ +/* Generic video mixer plugin + * + * GStreamer + * Copyright (C) 2015 Matthew Waters + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include + +#include "gstglbasemixer.h" + +#define gst_gl_base_mixer_parent_class parent_class +G_DEFINE_ABSTRACT_TYPE (GstGLBaseMixer, gst_gl_base_mixer, + GST_TYPE_VIDEO_AGGREGATOR); +static gboolean gst_gl_base_mixer_do_bufferpool (GstGLBaseMixer * mix, + GstCaps * outcaps); + +#define GST_CAT_DEFAULT gst_gl_base_mixer_debug +GST_DEBUG_CATEGORY (gst_gl_base_mixer_debug); + +static void gst_gl_base_mixer_pad_get_property (GObject * object, guint prop_id, + GValue * value, GParamSpec * pspec); +static void gst_gl_base_mixer_pad_set_property (GObject * object, guint prop_id, + const GValue * value, GParamSpec * pspec); +static void gst_gl_base_mixer_pad_finalize (GObject * object); + +static void gst_gl_base_mixer_set_context (GstElement * element, + GstContext * context); + +enum +{ + PROP_PAD_0 +}; + +#define GST_GL_BASE_MIXER_GET_PRIVATE(obj) \ + (G_TYPE_INSTANCE_GET_PRIVATE ((obj), GST_TYPE_GL_BASE_MIXER, GstGLBaseMixerPrivate)) + +struct _GstGLBaseMixerPrivate +{ + gboolean negotiated; + + GstGLContext *other_context; + + GstBufferPool *pool; + GstAllocator *allocator; + GstAllocationParams params; + GstQuery *query; +}; + +G_DEFINE_TYPE (GstGLBaseMixerPad, gst_gl_base_mixer_pad, + GST_TYPE_VIDEO_AGGREGATOR_PAD); + +static void +gst_gl_base_mixer_pad_class_init (GstGLBaseMixerPadClass * klass) +{ + GObjectClass *gobject_class = (GObjectClass *) klass; + GstVideoAggregatorPadClass *vaggpad_class = + (GstVideoAggregatorPadClass *) klass; + + gobject_class->set_property = gst_gl_base_mixer_pad_set_property; + gobject_class->get_property = gst_gl_base_mixer_pad_get_property; + + gobject_class->finalize = gst_gl_base_mixer_pad_finalize; + + vaggpad_class->set_info = NULL; + vaggpad_class->prepare_frame = NULL; + vaggpad_class->clean_frame = NULL; +} + +static void +gst_gl_base_mixer_pad_finalize (GObject * object) +{ + G_OBJECT_CLASS (gst_gl_base_mixer_pad_parent_class)->finalize (object); +} + +static void +gst_gl_base_mixer_pad_get_property (GObject * object, guint prop_id, + GValue * value, GParamSpec * pspec) +{ + switch (prop_id) { + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static void +gst_gl_base_mixer_pad_set_property (GObject * object, guint prop_id, + const GValue * value, GParamSpec * pspec) +{ + switch (prop_id) { + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static gboolean +_negotiated_caps (GstVideoAggregator * vagg, GstCaps * caps) +{ + GstGLBaseMixer *mix = GST_GL_BASE_MIXER (vagg); + + return gst_gl_base_mixer_do_bufferpool (mix, caps); +} + +static gboolean +_default_propose_allocation (GstGLBaseMixer * mix, GstGLBaseMixerPad * pad, + GstQuery * decide_query, GstQuery * query) +{ + return TRUE; +} + +static gboolean +gst_gl_base_mixer_sink_query (GstAggregator * agg, GstAggregatorPad * bpad, + GstQuery * query) +{ + gboolean ret = FALSE; + GstGLBaseMixer *mix = GST_GL_BASE_MIXER (agg); + GstGLBaseMixerClass *mix_class = GST_GL_BASE_MIXER_GET_CLASS (mix); + GstGLBaseMixerPad *pad = GST_GL_BASE_MIXER_PAD (bpad); + + GST_TRACE ("QUERY %" GST_PTR_FORMAT, query); + + switch (GST_QUERY_TYPE (query)) { + case GST_QUERY_ALLOCATION: + { + GstQuery *decide_query = NULL; + + GST_OBJECT_LOCK (mix); + if (G_UNLIKELY (!mix->priv->negotiated)) { + GST_DEBUG_OBJECT (mix, + "not negotiated yet, can't answer ALLOCATION query"); + GST_OBJECT_UNLOCK (mix); + return FALSE; + } + if ((decide_query = mix->priv->query)) + gst_query_ref (decide_query); + GST_OBJECT_UNLOCK (mix); + + GST_DEBUG_OBJECT (mix, + "calling propose allocation with query %" GST_PTR_FORMAT, + decide_query); + + /* pass the query to the propose_allocation vmethod if any */ + if (mix_class->propose_allocation) + ret = mix_class->propose_allocation (mix, pad, decide_query, query); + else + ret = FALSE; + + if (decide_query) + gst_query_unref (decide_query); + + GST_DEBUG_OBJECT (mix, "ALLOCATION ret %d, %" GST_PTR_FORMAT, ret, query); + return ret; + } + case GST_QUERY_CONTEXT: + { + const gchar *context_type; + GstContext *context, *old_context; + + ret = gst_gl_handle_context_query ((GstElement *) mix, query, + &mix->display, &mix->priv->other_context); + if (mix->display) + gst_gl_display_filter_gl_api (mix->display, + mix_class->supported_gl_api); + gst_query_parse_context_type (query, &context_type); + + if (g_strcmp0 (context_type, "gst.gl.local_context") == 0) { + GstStructure *s; + + gst_query_parse_context (query, &old_context); + + if (old_context) + context = gst_context_copy (old_context); + else + context = gst_context_new ("gst.gl.local_context", FALSE); + + s = gst_context_writable_structure (context); + gst_structure_set (s, "context", GST_GL_TYPE_CONTEXT, mix->context, + NULL); + gst_query_set_context (query, context); + gst_context_unref (context); + + ret = mix->context != NULL; + } + GST_LOG_OBJECT (mix, "context query of type %s %i", context_type, ret); + + if (ret) + return ret; + + break; + } + default: + break; + } + + return GST_AGGREGATOR_CLASS (parent_class)->sink_query (agg, bpad, query);; +} + +static void +gst_gl_base_mixer_pad_init (GstGLBaseMixerPad * mixerpad) +{ +} + +/* GLBaseMixer signals and args */ +enum +{ + /* FILL ME */ + LAST_SIGNAL +}; + +enum +{ + PROP_0, + PROP_CONTEXT +}; + +static gboolean gst_gl_base_mixer_src_query (GstAggregator * agg, + GstQuery * query); + +static gboolean +gst_gl_base_mixer_src_activate_mode (GstAggregator * aggregator, + GstPadMode mode, gboolean active); +static gboolean gst_gl_base_mixer_stop (GstAggregator * agg); +static gboolean gst_gl_base_mixer_start (GstAggregator * agg); + +static void gst_gl_base_mixer_set_property (GObject * object, guint prop_id, + const GValue * value, GParamSpec * pspec); +static void gst_gl_base_mixer_get_property (GObject * object, guint prop_id, + GValue * value, GParamSpec * pspec); + +static gboolean gst_gl_base_mixer_decide_allocation (GstGLBaseMixer * mix, + GstQuery * query); +static gboolean gst_gl_base_mixer_set_allocation (GstGLBaseMixer * mix, + GstBufferPool * pool, GstAllocator * allocator, + GstAllocationParams * params, GstQuery * query); + +static void gst_gl_base_mixer_finalize (GObject * object); + +static void +gst_gl_base_mixer_class_init (GstGLBaseMixerClass * klass) +{ + GObjectClass *gobject_class; + GstElementClass *element_class; + + GstVideoAggregatorClass *videoaggregator_class = + (GstVideoAggregatorClass *) klass; + GstAggregatorClass *agg_class = (GstAggregatorClass *) klass; + + GST_DEBUG_CATEGORY_INIT (GST_CAT_DEFAULT, "glmixer", 0, "opengl mixer"); + + gobject_class = (GObjectClass *) klass; + element_class = GST_ELEMENT_CLASS (klass); + + g_type_class_add_private (klass, sizeof (GstGLBaseMixerPrivate)); + + gobject_class->finalize = GST_DEBUG_FUNCPTR (gst_gl_base_mixer_finalize); + + gobject_class->get_property = gst_gl_base_mixer_get_property; + gobject_class->set_property = gst_gl_base_mixer_set_property; + + element_class->set_context = + GST_DEBUG_FUNCPTR (gst_gl_base_mixer_set_context); + + agg_class->sinkpads_type = GST_TYPE_GL_BASE_MIXER_PAD; + agg_class->sink_query = gst_gl_base_mixer_sink_query; + agg_class->src_query = gst_gl_base_mixer_src_query; + agg_class->src_activate = gst_gl_base_mixer_src_activate_mode; + agg_class->stop = gst_gl_base_mixer_stop; + agg_class->start = gst_gl_base_mixer_start; + + videoaggregator_class->negotiated_caps = _negotiated_caps; + + klass->propose_allocation = _default_propose_allocation; + + g_object_class_install_property (gobject_class, PROP_CONTEXT, + g_param_spec_object ("context", + "OpenGL context", + "Get OpenGL context", + GST_GL_TYPE_CONTEXT, G_PARAM_READABLE | G_PARAM_STATIC_STRINGS)); + + /* Register the pad class */ + g_type_class_ref (GST_TYPE_GL_BASE_MIXER_PAD); + + klass->supported_gl_api = GST_GL_API_ANY; +} + +static void +gst_gl_base_mixer_reset (GstGLBaseMixer * mix) +{ + /* clean up collect data */ + mix->priv->negotiated = FALSE; +} + +static void +gst_gl_base_mixer_init (GstGLBaseMixer * mix) +{ + mix->priv = GST_GL_BASE_MIXER_GET_PRIVATE (mix); + + gst_gl_base_mixer_reset (mix); +} + +static void +gst_gl_base_mixer_finalize (GObject * object) +{ + GstGLBaseMixer *mix = GST_GL_BASE_MIXER (object); + + if (mix->priv->other_context) { + gst_object_unref (mix->priv->other_context); + mix->priv->other_context = NULL; + } + + G_OBJECT_CLASS (parent_class)->finalize (object); +} + +static void +gst_gl_base_mixer_set_context (GstElement * element, GstContext * context) +{ + GstGLBaseMixer *mix = GST_GL_BASE_MIXER (element); + GstGLBaseMixerClass *mix_class = GST_GL_BASE_MIXER_GET_CLASS (mix); + + gst_gl_handle_set_context (element, context, &mix->display, + &mix->priv->other_context); + + if (mix->display) + gst_gl_display_filter_gl_api (mix->display, mix_class->supported_gl_api); +} + +static gboolean +gst_gl_base_mixer_activate (GstGLBaseMixer * mix, gboolean active) +{ + GstGLBaseMixerClass *mix_class = GST_GL_BASE_MIXER_GET_CLASS (mix); + gboolean result = TRUE; + + if (active) { + if (!gst_gl_ensure_element_data (mix, &mix->display, + &mix->priv->other_context)) + return FALSE; + + gst_gl_display_filter_gl_api (mix->display, mix_class->supported_gl_api); + } + + return result; +} + +static gboolean +gst_gl_base_mixer_src_activate_mode (GstAggregator * aggregator, + GstPadMode mode, gboolean active) +{ + GstGLBaseMixer *mix; + gboolean result = FALSE; + + mix = GST_GL_BASE_MIXER (aggregator); + + switch (mode) { + case GST_PAD_MODE_PUSH: + case GST_PAD_MODE_PULL: + result = gst_gl_base_mixer_activate (mix, active); + break; + default: + result = TRUE; + break; + } + return result; +} + +static gboolean +gst_gl_base_mixer_src_query (GstAggregator * agg, GstQuery * query) +{ + gboolean res = FALSE; + GstGLBaseMixer *mix = GST_GL_BASE_MIXER (agg); + GstGLBaseMixerClass *mix_class = GST_GL_BASE_MIXER_GET_CLASS (mix); + + switch (GST_QUERY_TYPE (query)) { + case GST_QUERY_CONTEXT: + { + const gchar *context_type; + GstContext *context, *old_context; + + res = gst_gl_handle_context_query ((GstElement *) mix, query, + &mix->display, &mix->priv->other_context); + if (mix->display) + gst_gl_display_filter_gl_api (mix->display, + mix_class->supported_gl_api); + gst_query_parse_context_type (query, &context_type); + + if (g_strcmp0 (context_type, "gst.gl.local_context") == 0) { + GstStructure *s; + + gst_query_parse_context (query, &old_context); + + if (old_context) + context = gst_context_copy (old_context); + else + context = gst_context_new ("gst.gl.local_context", FALSE); + + s = gst_context_writable_structure (context); + gst_structure_set (s, "context", GST_GL_TYPE_CONTEXT, mix->context, + NULL); + gst_query_set_context (query, context); + gst_context_unref (context); + + res = mix->context != NULL; + } + GST_LOG_OBJECT (mix, "context query of type %s %i", context_type, res); + + if (res) + return res; + + break; + } + default: + break; + } + + return GST_AGGREGATOR_CLASS (parent_class)->src_query (agg, query); +} + +static gboolean +_find_local_gl_context (GstGLBaseMixer * mix) +{ + GstQuery *query; + GstContext *context; + const GstStructure *s; + + if (mix->context) + return TRUE; + + query = gst_query_new_context ("gst.gl.local_context"); + if (!mix->context && gst_gl_run_query (GST_ELEMENT (mix), query, GST_PAD_SRC)) { + gst_query_parse_context (query, &context); + if (context) { + s = gst_context_get_structure (context); + gst_structure_get (s, "context", GST_GL_TYPE_CONTEXT, &mix->context, + NULL); + } + } + if (!mix->context + && gst_gl_run_query (GST_ELEMENT (mix), query, GST_PAD_SINK)) { + gst_query_parse_context (query, &context); + if (context) { + s = gst_context_get_structure (context); + gst_structure_get (s, "context", GST_GL_TYPE_CONTEXT, &mix->context, + NULL); + } + } + + GST_DEBUG_OBJECT (mix, "found local context %p", mix->context); + + gst_query_unref (query); + + if (mix->context) + return TRUE; + + return FALSE; +} + +static gboolean +gst_gl_base_mixer_decide_allocation (GstGLBaseMixer * mix, GstQuery * query) +{ + GstGLBaseMixerClass *mix_class = GST_GL_BASE_MIXER_GET_CLASS (mix); + GError *error = NULL; + guint idx; + GstGLContext *other_context = NULL; + gboolean ret = TRUE; + + if (!gst_gl_ensure_element_data (mix, &mix->display, + &mix->priv->other_context)) + return FALSE; + + gst_gl_display_filter_gl_api (mix->display, mix_class->supported_gl_api); + + _find_local_gl_context (mix); + + if (gst_query_find_allocation_meta (query, + GST_VIDEO_GL_TEXTURE_UPLOAD_META_API_TYPE, &idx)) { + GstGLContext *context; + const GstStructure *upload_meta_params; + gpointer handle; + gchar *type; + gchar *apis; + + gst_query_parse_nth_allocation_meta (query, idx, &upload_meta_params); + if (upload_meta_params) { + if (gst_structure_get (upload_meta_params, "gst.gl.GstGLContext", + GST_GL_TYPE_CONTEXT, &context, NULL) && context) { + GstGLContext *old = mix->context; + + mix->context = context; + if (old) + gst_object_unref (old); + } else if (gst_structure_get (upload_meta_params, "gst.gl.context.handle", + G_TYPE_POINTER, &handle, "gst.gl.context.type", G_TYPE_STRING, + &type, "gst.gl.context.apis", G_TYPE_STRING, &apis, NULL) + && handle) { + GstGLPlatform platform; + GstGLAPI gl_apis; + + GST_DEBUG ("got GL context handle 0x%p with type %s and apis %s", + handle, type, apis); + + platform = gst_gl_platform_from_string (type); + gl_apis = gst_gl_api_from_string (apis); + + if (gl_apis && platform) + other_context = + gst_gl_context_new_wrapped (mix->display, (guintptr) handle, + platform, gl_apis); + } + } + } + + if (mix->priv->other_context) { + if (!other_context) { + other_context = mix->priv->other_context; + } else { + GST_ELEMENT_WARNING (mix, LIBRARY, SETTINGS, + ("%s", "Cannot share with more than one GL context"), + ("%s", "Cannot share with more than one GL context")); + } + } + + if (!mix->context) { + mix->context = gst_gl_context_new (mix->display); + if (!gst_gl_context_create (mix->context, other_context, &error)) + goto context_error; + } + + if (mix_class->decide_allocation) + ret = mix_class->decide_allocation (mix, query); + + return ret; + +context_error: + { + GST_ELEMENT_ERROR (mix, RESOURCE, NOT_FOUND, ("%s", error->message), + (NULL)); + return FALSE; + } +} + +/* takes ownership of the pool, allocator and query */ +static gboolean +gst_gl_base_mixer_set_allocation (GstGLBaseMixer * mix, + GstBufferPool * pool, GstAllocator * allocator, + GstAllocationParams * params, GstQuery * query) +{ + GstAllocator *oldalloc; + GstBufferPool *oldpool; + GstQuery *oldquery; + GstGLBaseMixerPrivate *priv = mix->priv; + + GST_DEBUG ("storing allocation query"); + + GST_OBJECT_LOCK (mix); + oldpool = priv->pool; + priv->pool = pool; + + oldalloc = priv->allocator; + priv->allocator = allocator; + + oldquery = priv->query; + priv->query = query; + + if (params) + priv->params = *params; + else + gst_allocation_params_init (&priv->params); + GST_OBJECT_UNLOCK (mix); + + if (oldpool) { + GST_DEBUG_OBJECT (mix, "deactivating old pool %p", oldpool); + gst_buffer_pool_set_active (oldpool, FALSE); + gst_object_unref (oldpool); + } + if (oldalloc) { + gst_object_unref (oldalloc); + } + if (oldquery) { + gst_query_unref (oldquery); + } + return TRUE; +} + +static gboolean +gst_gl_base_mixer_do_bufferpool (GstGLBaseMixer * mix, GstCaps * outcaps) +{ + GstQuery *query; + gboolean result = TRUE; + GstBufferPool *pool = NULL; + GstAllocator *allocator; + GstAllocationParams params; + GstAggregator *agg = GST_AGGREGATOR (mix); + + /* find a pool for the negotiated caps now */ + GST_DEBUG_OBJECT (mix, "doing allocation query"); + query = gst_query_new_allocation (outcaps, TRUE); + if (!gst_pad_peer_query (agg->srcpad, query)) { + /* not a problem, just debug a little */ + GST_DEBUG_OBJECT (mix, "peer ALLOCATION query failed"); + } + + GST_DEBUG_OBJECT (mix, "calling decide_allocation"); + result = gst_gl_base_mixer_decide_allocation (mix, query); + + GST_DEBUG_OBJECT (mix, "ALLOCATION (%d) params: %" GST_PTR_FORMAT, result, + query); + + if (!result) + goto no_decide_allocation; + + /* we got configuration from our peer or the decide_allocation method, + * parse them */ + if (gst_query_get_n_allocation_params (query) > 0) { + gst_query_parse_nth_allocation_param (query, 0, &allocator, ¶ms); + } else { + allocator = NULL; + gst_allocation_params_init (¶ms); + } + + if (gst_query_get_n_allocation_pools (query) > 0) + gst_query_parse_nth_allocation_pool (query, 0, &pool, NULL, NULL, NULL); + + /* now store */ + result = + gst_gl_base_mixer_set_allocation (mix, pool, allocator, ¶ms, query); + + return result; + + /* Errors */ +no_decide_allocation: + { + GST_WARNING_OBJECT (mix, "Failed to decide allocation"); + gst_query_unref (query); + + return result; + } +} + +GstBufferPool * +gst_gl_base_mixer_get_buffer_pool (GstGLBaseMixer * mix) +{ + GstBufferPool *pool; + + GST_OBJECT_LOCK (mix); + pool = mix->priv->pool; + if (pool) + gst_object_ref (pool); + GST_OBJECT_UNLOCK (mix); + + return pool; +} + +static void +gst_gl_base_mixer_get_property (GObject * object, + guint prop_id, GValue * value, GParamSpec * pspec) +{ + GstGLBaseMixer *mixer = GST_GL_BASE_MIXER (object); + + switch (prop_id) { + case PROP_CONTEXT: + g_value_set_object (value, mixer->context); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static void +gst_gl_base_mixer_set_property (GObject * object, + guint prop_id, const GValue * value, GParamSpec * pspec) +{ + switch (prop_id) { + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static gboolean +gst_gl_base_mixer_start (GstAggregator * agg) +{ + return TRUE; +} + +static gboolean +gst_gl_base_mixer_stop (GstAggregator * agg) +{ + GstGLBaseMixer *mix = GST_GL_BASE_MIXER (agg); + + if (mix->priv->query) { + gst_query_unref (mix->priv->query); + mix->priv->query = NULL; + } + + if (mix->priv->pool) { + gst_object_unref (mix->priv->pool); + mix->priv->pool = NULL; + } + + if (mix->display) { + gst_object_unref (mix->display); + mix->display = NULL; + } + + if (mix->context) { + gst_object_unref (mix->context); + mix->context = NULL; + } + + gst_gl_base_mixer_reset (mix); + + return TRUE; +} diff --git a/ext/gl/gstglbasemixer.h b/ext/gl/gstglbasemixer.h new file mode 100644 index 0000000000..6282ccb96e --- /dev/null +++ b/ext/gl/gstglbasemixer.h @@ -0,0 +1,103 @@ +/* + * GStreamer + * Copyright (C) 2009 Julien Isorce + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifndef __GST_GL_BASE_MIXER_H__ +#define __GST_GL_BASE_MIXER_H__ + +#include +#include +#include +#include + +G_BEGIN_DECLS + +#define GST_TYPE_GL_BASE_MIXER_PAD (gst_gl_base_mixer_pad_get_type()) +#define GST_GL_BASE_MIXER_PAD(obj) \ + (G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_GL_BASE_MIXER_PAD, GstGLBaseMixerPad)) +#define GST_GL_BASE_MIXER_PAD_CLASS(klass) \ + (G_TYPE_CHECK_CLASS_CAST((klass),GST_TYPE_GL_BASE_MIXER_PAD, GstGLBaseMixerPadClass)) +#define GST_IS_GL_BASE_MIXER_PAD(obj) \ + (G_TYPE_CHECK_INSTANCE_TYPE((obj),GST_TYPE_GL_BASE_MIXER_PAD)) +#define GST_IS_GL_BASE_MIXER_PAD_CLASS(klass) \ + (G_TYPE_CHECK_CLASS_TYPE((klass),GST_TYPE_GL_BASE_MIXER_PAD)) +#define GST_GL_BASE_MIXER_PAD_GET_CLASS(obj) \ + (G_TYPE_INSTANCE_GET_CLASS((obj),GST_TYPE_GL_BASE_MIXER_PAD,GstGLBaseMixerPadClass)) + +typedef struct _GstGLBaseMixerPad GstGLBaseMixerPad; +typedef struct _GstGLBaseMixerPadClass GstGLBaseMixerPadClass; + +/* all information needed for one video stream */ +struct _GstGLBaseMixerPad +{ + GstVideoAggregatorPad parent; /* subclass the pad */ +}; + +struct _GstGLBaseMixerPadClass +{ + GstVideoAggregatorPadClass parent_class; +}; + +GType gst_gl_base_mixer_pad_get_type (void); + +#define GST_TYPE_GL_BASE_MIXER (gst_gl_base_mixer_get_type()) +#define GST_GL_BASE_MIXER(obj) \ + (G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_GL_BASE_MIXER, GstGLBaseMixer)) +#define GST_GL_BASE_MIXER_CLASS(klass) \ + (G_TYPE_CHECK_CLASS_CAST((klass),GST_TYPE_GL_BASE_MIXER, GstGLBaseMixerClass)) +#define GST_IS_GL_BASE_MIXER(obj) \ + (G_TYPE_CHECK_INSTANCE_TYPE((obj),GST_TYPE_GL_BASE_MIXER)) +#define GST_IS_GL_BASE_MIXER_CLASS(klass) \ + (G_TYPE_CHECK_CLASS_TYPE((klass),GST_TYPE_GL_BASE_MIXER)) +#define GST_GL_BASE_MIXER_GET_CLASS(obj) \ + (G_TYPE_INSTANCE_GET_CLASS((obj),GST_TYPE_GL_BASE_MIXER,GstGLBaseMixerClass)) + +typedef struct _GstGLBaseMixer GstGLBaseMixer; +typedef struct _GstGLBaseMixerClass GstGLBaseMixerClass; +typedef struct _GstGLBaseMixerPrivate GstGLBaseMixerPrivate; + +struct _GstGLBaseMixer +{ + GstVideoAggregator vaggregator; + + GstGLDisplay *display; + GstGLContext *context; + + gpointer _padding[GST_PADDING]; + + GstGLBaseMixerPrivate *priv; +}; + +struct _GstGLBaseMixerClass +{ + GstVideoAggregatorClass parent_class; + GstGLAPI supported_gl_api; + + gboolean (*propose_allocation) (GstGLBaseMixer * mix, GstGLBaseMixerPad * pad, GstQuery * decide_query, GstQuery *query); + gboolean (*decide_allocation) (GstGLBaseMixer * mix, GstQuery * decide_query); + + gpointer _padding[GST_PADDING]; +}; + +GType gst_gl_base_mixer_get_type(void); + +GstBufferPool *gst_gl_base_mixer_get_buffer_pool (GstGLBaseMixer * mix); + +G_END_DECLS +#endif /* __GST_GL_BASE_MIXER_H__ */ diff --git a/ext/gl/gstglmixer.c b/ext/gl/gstglmixer.c index 82a685f765..633a11e316 100644 --- a/ext/gl/gstglmixer.c +++ b/ext/gl/gstglmixer.c @@ -24,16 +24,8 @@ #endif #include -#include #include -#ifdef HAVE_STDLIB_H -#include -#endif -#ifdef HAVE_STRING_H -#include -#endif - #include "gstglmixer.h" #if GST_GL_HAVE_PLATFORM_EGL @@ -41,10 +33,7 @@ #endif #define gst_gl_mixer_parent_class parent_class -G_DEFINE_ABSTRACT_TYPE (GstGLMixer, gst_gl_mixer, GST_TYPE_VIDEO_AGGREGATOR); -static gboolean gst_gl_mixer_do_bufferpool (GstGLMixer * mix, - GstCaps * outcaps); - +G_DEFINE_ABSTRACT_TYPE (GstGLMixer, gst_gl_mixer, GST_TYPE_GL_BASE_MIXER); #define GST_CAT_DEFAULT gst_gl_mixer_debug GST_DEBUG_CATEGORY (gst_gl_mixer_debug); @@ -57,9 +46,6 @@ static void gst_gl_mixer_pad_finalize (GObject * object); static GstBuffer *_default_pad_upload_buffer (GstGLMixer * mix, GstGLMixerFrameData * frame, GstBuffer * buffer); -static void gst_gl_mixer_set_context (GstElement * element, - GstContext * context); - enum { PROP_PAD_0 @@ -72,18 +58,12 @@ struct _GstGLMixerPrivate { gboolean negotiated; - GstBufferPool *pool; - gboolean pool_active; - GstAllocator *allocator; - GstAllocationParams params; - GstQuery *query; - gboolean gl_resource_ready; GMutex gl_resource_lock; GCond gl_resource_cond; }; -G_DEFINE_TYPE (GstGLMixerPad, gst_gl_mixer_pad, GST_TYPE_VIDEO_AGGREGATOR_PAD); +G_DEFINE_TYPE (GstGLMixerPad, gst_gl_mixer_pad, GST_TYPE_GL_BASE_MIXER_PAD); static void gst_gl_mixer_pad_class_init (GstGLMixerPadClass * klass) @@ -121,6 +101,7 @@ static void _init_upload (GstGLMixer * mix, GstGLMixerPad * pad) { GstVideoAggregatorPad *vaggpad = GST_VIDEO_AGGREGATOR_PAD (pad); + GstGLContext *context = GST_GL_BASE_MIXER (mix)->context; if (!pad->upload) { GstCaps *in_caps = gst_pad_get_current_caps (GST_PAD (pad)); @@ -128,7 +109,7 @@ _init_upload (GstGLMixer * mix, GstGLMixerPad * pad) GstCapsFeatures *gl_features = gst_caps_features_from_string (GST_CAPS_FEATURE_MEMORY_GL_MEMORY); - pad->upload = gst_gl_upload_new (mix->context); + pad->upload = gst_gl_upload_new (context); gst_caps_set_features (upload_caps, 0, gst_caps_features_copy (gl_features)); @@ -146,7 +127,7 @@ _init_upload (GstGLMixer * mix, GstGLMixerPad * pad) gl_caps = gst_video_info_to_caps (&gl_info); gst_caps_set_features (gl_caps, 0, gst_caps_features_copy (gl_features)); - pad->convert = gst_gl_color_convert_new (mix->context); + pad->convert = gst_gl_color_convert_new (context); gst_gl_color_convert_set_caps (pad->convert, upload_caps, gl_caps); gst_caps_unref (gl_caps); @@ -183,44 +164,40 @@ static gboolean _negotiated_caps (GstVideoAggregator * vagg, GstCaps * caps) { GstGLMixer *mix = GST_GL_MIXER (vagg); - gboolean ret = gst_gl_mixer_do_bufferpool (mix, caps); + GstGLMixerClass *mix_class = GST_GL_MIXER_GET_CLASS (mix); + GstGLBaseMixerClass *base_mix_class = GST_GL_BASE_MIXER_GET_CLASS (mix); + gboolean ret; - mix->priv->negotiated = ret; + mix->priv->negotiated = TRUE; + base_mix_class->supported_gl_api = mix_class->supported_gl_api; gst_caps_replace (&mix->out_caps, caps); + ret = GST_VIDEO_AGGREGATOR_CLASS (parent_class)->negotiated_caps (vagg, caps); + + mix->context = GST_GL_BASE_MIXER (mix)->context; + return ret; } static gboolean -gst_gl_mixer_propose_allocation (GstGLMixer * mix, GstGLMixerPad * pad, - GstQuery * decide_query, GstQuery * query) +gst_gl_mixer_propose_allocation (GstGLBaseMixer * base_mix, + GstGLBaseMixerPad * base_pad, GstQuery * decide_query, GstQuery * query) { - GstGLMixerClass *mix_class = GST_GL_MIXER_GET_CLASS (mix); + GstGLMixer *mix = GST_GL_MIXER (base_mix); + GstGLMixerPad *pad = GST_GL_MIXER_PAD (base_pad); + GstGLContext *context = base_mix->context; GstBufferPool *pool = NULL; GstStructure *config; GstCaps *caps; guint size = 0; gboolean need_pool; - GError *error = NULL; gst_query_parse_allocation (query, &caps, &need_pool); if (caps == NULL) goto no_caps; - /* FIXME this is not thread safe, this method can be called in parallel */ - if (!gst_gl_ensure_element_data (mix, &mix->display, &mix->other_context)) - return FALSE; - - gst_gl_display_filter_gl_api (mix->display, mix_class->supported_gl_api); - - if (!mix->context) { - mix->context = gst_gl_context_new (mix->display); - if (!gst_gl_context_create (mix->context, mix->other_context, &error)) - goto context_error; - } - if (need_pool) { GstVideoInfo info; @@ -228,7 +205,7 @@ gst_gl_mixer_propose_allocation (GstGLMixer * mix, GstGLMixerPad * pad, goto invalid_caps; GST_DEBUG_OBJECT (mix, "create new pool"); - pool = gst_gl_buffer_pool_new (mix->context); + pool = gst_gl_buffer_pool_new (context); /* the normal size of a frame */ size = info.size; @@ -245,7 +222,7 @@ gst_gl_mixer_propose_allocation (GstGLMixer * mix, GstGLMixerPad * pad, } /* we also support various metadata */ - if (mix->context->gl_vtable->FenceSync) + if (context->gl_vtable->FenceSync) gst_query_add_allocation_meta (query, GST_GL_SYNC_META_API_TYPE, 0); _init_upload (mix, pad); @@ -270,12 +247,6 @@ config_failed: GST_DEBUG_OBJECT (mix, "failed setting config"); return FALSE; } -context_error: - { - GST_ELEMENT_ERROR (mix, RESOURCE, NOT_FOUND, ("%s", error->message), - (NULL)); - return FALSE; - } } static gboolean @@ -310,15 +281,17 @@ gst_gl_mixer_set_caps_features (const GstCaps * caps, GstCaps * gst_gl_mixer_update_caps (GstGLMixer * mix, GstCaps * caps) { - GstCaps *result, *tmp; + GstGLContext *context = GST_GL_BASE_MIXER (mix)->context; + GstCaps *result, *tmp, *gl_caps; + + gl_caps = gst_caps_from_string ("video/x-raw(memory:GLMemory),format=RGBA"); result = - gst_gl_color_convert_transform_caps (mix->context, GST_PAD_SRC, caps, - NULL); + gst_gl_color_convert_transform_caps (context, GST_PAD_SRC, gl_caps, NULL); tmp = result; GST_DEBUG_OBJECT (mix, "convert returned caps %" GST_PTR_FORMAT, tmp); - result = gst_gl_upload_transform_caps (mix->context, GST_PAD_SRC, tmp, NULL); + result = gst_gl_upload_transform_caps (context, GST_PAD_SRC, tmp, NULL); gst_caps_unref (tmp); tmp = result; GST_DEBUG_OBJECT (mix, "transfer returned caps %" GST_PTR_FORMAT, tmp); @@ -367,8 +340,6 @@ gst_gl_mixer_sink_query (GstAggregator * agg, GstAggregatorPad * bpad, { gboolean ret = FALSE; GstGLMixer *mix = GST_GL_MIXER (agg); - GstGLMixerClass *mix_class = GST_GL_MIXER_GET_CLASS (mix); - GstGLMixerPad *pad = GST_GL_MIXER_PAD (bpad); GST_TRACE ("QUERY %" GST_PTR_FORMAT, query); @@ -394,43 +365,6 @@ gst_gl_mixer_sink_query (GstAggregator * agg, GstAggregatorPad * bpad, ret = TRUE; break; } - case GST_QUERY_ALLOCATION: - { - GstQuery *decide_query = NULL; - - GST_OBJECT_LOCK (mix); - if (G_UNLIKELY (!mix->priv->negotiated)) { - GST_DEBUG_OBJECT (mix, - "not negotiated yet, can't answer ALLOCATION query"); - GST_OBJECT_UNLOCK (mix); - return FALSE; - } - if ((decide_query = mix->priv->query)) - gst_query_ref (decide_query); - GST_OBJECT_UNLOCK (mix); - - GST_DEBUG_OBJECT (mix, - "calling propose allocation with query %" GST_PTR_FORMAT, - decide_query); - - /* pass the query to the propose_allocation vmethod if any */ - ret = gst_gl_mixer_propose_allocation (mix, pad, decide_query, query); - - if (decide_query) - gst_query_unref (decide_query); - - GST_DEBUG_OBJECT (mix, "ALLOCATION ret %d, %" GST_PTR_FORMAT, ret, query); - break; - } - case GST_QUERY_CONTEXT: - { - ret = gst_gl_handle_context_query ((GstElement *) mix, query, - &mix->display, &mix->other_context); - if (mix->display) - gst_gl_display_filter_gl_api (mix->display, - mix_class->supported_gl_api); - break; - } default: ret = GST_AGGREGATOR_CLASS (parent_class)->sink_query (agg, bpad, query); break; @@ -454,7 +388,6 @@ enum enum { PROP_0, - PROP_CONTEXT }; static GstStaticPadTemplate src_factory = GST_STATIC_PAD_TEMPLATE ("src", @@ -486,12 +419,8 @@ static GstStaticPadTemplate sink_factory = GST_STATIC_PAD_TEMPLATE ("sink_%u", ); static gboolean gst_gl_mixer_src_query (GstAggregator * agg, GstQuery * query); -static GstFlowReturn -gst_gl_mixer_get_output_buffer (GstVideoAggregator * videoaggregator, - GstBuffer ** outbuf); -static gboolean -gst_gl_mixer_src_activate_mode (GstAggregator * aggregator, GstPadMode mode, - gboolean active); +static GstFlowReturn gst_gl_mixer_get_output_buffer (GstVideoAggregator * + videoaggregator, GstBuffer ** outbuf); static gboolean gst_gl_mixer_stop (GstAggregator * agg); static gboolean gst_gl_mixer_start (GstAggregator * agg); @@ -504,28 +433,22 @@ static void gst_gl_mixer_set_property (GObject * object, guint prop_id, static void gst_gl_mixer_get_property (GObject * object, guint prop_id, GValue * value, GParamSpec * pspec); -static gboolean gst_gl_mixer_decide_allocation (GstGLMixer * mix, +static gboolean gst_gl_mixer_decide_allocation (GstGLBaseMixer * mix, GstQuery * query); -static gboolean gst_gl_mixer_set_allocation (GstGLMixer * mix, - GstBufferPool * pool, GstAllocator * allocator, - GstAllocationParams * params, GstQuery * query); static void gst_gl_mixer_finalize (GObject * object); static void gst_gl_mixer_class_init (GstGLMixerClass * klass) { - GObjectClass *gobject_class; - GstElementClass *element_class; - + GObjectClass *gobject_class = (GObjectClass *) klass; + GstElementClass *element_class = GST_ELEMENT_CLASS (klass); GstVideoAggregatorClass *videoaggregator_class = (GstVideoAggregatorClass *) klass; GstAggregatorClass *agg_class = (GstAggregatorClass *) klass; + GstGLBaseMixerClass *mix_class = GST_GL_BASE_MIXER_CLASS (klass);; - GST_DEBUG_CATEGORY_INIT (GST_CAT_DEFAULT, "glmixer", 0, "opengl mixer"); - - gobject_class = (GObjectClass *) klass; - element_class = GST_ELEMENT_CLASS (klass); + GST_DEBUG_CATEGORY_INIT (GST_CAT_DEFAULT, "glmixer", 0, "OpenGL mixer"); g_type_class_add_private (klass, sizeof (GstGLMixerPrivate)); @@ -539,12 +462,9 @@ gst_gl_mixer_class_init (GstGLMixerClass * klass) gst_element_class_add_pad_template (element_class, gst_static_pad_template_get (&sink_factory)); - element_class->set_context = GST_DEBUG_FUNCPTR (gst_gl_mixer_set_context); - agg_class->sinkpads_type = GST_TYPE_GL_MIXER_PAD; agg_class->sink_query = gst_gl_mixer_sink_query; agg_class->src_query = gst_gl_mixer_src_query; - agg_class->src_activate = gst_gl_mixer_src_activate_mode; agg_class->stop = gst_gl_mixer_stop; agg_class->start = gst_gl_mixer_start; @@ -553,11 +473,8 @@ gst_gl_mixer_class_init (GstGLMixerClass * klass) videoaggregator_class->negotiated_caps = _negotiated_caps; videoaggregator_class->find_best_format = NULL; - g_object_class_install_property (gobject_class, PROP_CONTEXT, - g_param_spec_object ("context", - "OpenGL context", - "Get OpenGL context", - GST_GL_TYPE_CONTEXT, G_PARAM_READABLE | G_PARAM_STATIC_STRINGS)); + mix_class->propose_allocation = gst_gl_mixer_propose_allocation; + mix_class->decide_allocation = gst_gl_mixer_decide_allocation; /* Register the pad class */ g_type_class_ref (GST_TYPE_GL_MIXER_PAD); @@ -569,7 +486,6 @@ gst_gl_mixer_class_init (GstGLMixerClass * klass) static void gst_gl_mixer_reset (GstGLMixer * mix) { - /* clean up collect data */ mix->priv->negotiated = FALSE; } @@ -578,7 +494,6 @@ gst_gl_mixer_init (GstGLMixer * mix) { mix->priv = GST_GL_MIXER_GET_PRIVATE (mix); mix->array_buffers = 0; - mix->display = NULL; mix->fbo = 0; mix->depthbuffer = 0; @@ -595,70 +510,16 @@ gst_gl_mixer_finalize (GObject * object) GstGLMixer *mix = GST_GL_MIXER (object); GstGLMixerPrivate *priv = mix->priv; - if (mix->other_context) { - gst_object_unref (mix->other_context); - mix->other_context = NULL; - } - g_mutex_clear (&priv->gl_resource_lock); g_cond_clear (&priv->gl_resource_cond); G_OBJECT_CLASS (parent_class)->finalize (object); } -static void -gst_gl_mixer_set_context (GstElement * element, GstContext * context) -{ - GstGLMixer *mix = GST_GL_MIXER (element); - GstGLMixerClass *mix_class = GST_GL_MIXER_GET_CLASS (mix); - - gst_gl_handle_set_context (element, context, &mix->display, - &mix->other_context); - - if (mix->display) - gst_gl_display_filter_gl_api (mix->display, mix_class->supported_gl_api); -} - -static gboolean -gst_gl_mixer_activate (GstGLMixer * mix, gboolean active) -{ - GstGLMixerClass *mix_class = GST_GL_MIXER_GET_CLASS (mix); - gboolean result = TRUE; - - if (active) { - if (!gst_gl_ensure_element_data (mix, &mix->display, &mix->other_context)) - return FALSE; - - gst_gl_display_filter_gl_api (mix->display, mix_class->supported_gl_api); - } - - return result; -} - -static gboolean -gst_gl_mixer_src_activate_mode (GstAggregator * aggregator, GstPadMode mode, - gboolean active) -{ - GstGLMixer *mix; - gboolean result = FALSE; - - mix = GST_GL_MIXER (aggregator); - - switch (mode) { - case GST_PAD_MODE_PUSH: - case GST_PAD_MODE_PULL: - result = gst_gl_mixer_activate (mix, active); - break; - default: - result = TRUE; - break; - } - return result; -} - static gboolean gst_gl_mixer_query_caps (GstPad * pad, GstAggregator * agg, GstQuery * query) { GstGLMixer *mix = GST_GL_MIXER (agg); + GstGLContext *context = GST_GL_BASE_MIXER (mix)->context; GstCaps *filter, *current_caps, *retcaps, *gl_caps; gst_query_parse_caps (query, &filter); @@ -674,7 +535,7 @@ gst_gl_mixer_query_caps (GstPad * pad, GstAggregator * agg, GstQuery * query) gst_gl_mixer_set_caps_features (current_caps, GST_CAPS_FEATURE_META_GST_VIDEO_GL_TEXTURE_UPLOAD_META)); retcaps = - gst_gl_download_transform_caps (mix->context, GST_PAD_SINK, current_caps, + gst_gl_download_transform_caps (context, GST_PAD_SINK, current_caps, NULL); retcaps = gst_caps_merge (gl_caps, retcaps); gst_caps_unref (current_caps); @@ -696,19 +557,8 @@ static gboolean gst_gl_mixer_src_query (GstAggregator * agg, GstQuery * query) { gboolean res = FALSE; - GstGLMixer *mix = GST_GL_MIXER (agg); - GstGLMixerClass *mix_class = GST_GL_MIXER_GET_CLASS (mix); switch (GST_QUERY_TYPE (query)) { - case GST_QUERY_CONTEXT: - { - res = gst_gl_handle_context_query ((GstElement *) mix, query, - &mix->display, &mix->other_context); - if (mix->display) - gst_gl_display_filter_gl_api (mix->display, - mix_class->supported_gl_api); - break; - } case GST_QUERY_CAPS: res = gst_gl_mixer_query_caps (agg->srcpad, agg, query); break; @@ -725,94 +575,43 @@ gst_gl_mixer_get_output_buffer (GstVideoAggregator * videoaggregator, GstBuffer ** outbuf) { GstGLMixer *mix = GST_GL_MIXER (videoaggregator); + GstBufferPool *pool; + GstFlowReturn ret; - if (!mix->priv->pool_active) { - if (!gst_buffer_pool_set_active (mix->priv->pool, TRUE)) { + pool = + gst_gl_base_mixer_get_buffer_pool (GST_GL_BASE_MIXER (videoaggregator)); + + if (!pool) + return GST_FLOW_NOT_NEGOTIATED; + + if (!gst_buffer_pool_is_active (pool)) { + if (!gst_buffer_pool_set_active (pool, TRUE)) { GST_ELEMENT_ERROR (mix, RESOURCE, SETTINGS, ("failed to activate bufferpool"), ("failed to activate bufferpool")); return GST_FLOW_ERROR; } - mix->priv->pool_active = TRUE; } - return gst_buffer_pool_acquire_buffer (mix->priv->pool, outbuf, NULL); + ret = gst_buffer_pool_acquire_buffer (pool, outbuf, NULL); + gst_object_unref (pool); + + return ret; } static gboolean -gst_gl_mixer_decide_allocation (GstGLMixer * mix, GstQuery * query) +gst_gl_mixer_decide_allocation (GstGLBaseMixer * base_mix, GstQuery * query) { + GstGLMixer *mix = GST_GL_MIXER (base_mix); GstGLMixerClass *mixer_class = GST_GL_MIXER_GET_CLASS (mix); + GstVideoAggregator *vagg = GST_VIDEO_AGGREGATOR (mix); + GstGLContext *context = base_mix->context; GstBufferPool *pool = NULL; GstStructure *config; GstCaps *caps; guint min, max, size; gboolean update_pool; GError *error = NULL; - guint idx; guint out_width, out_height; - GstGLContext *other_context = NULL; - GstVideoAggregator *vagg = GST_VIDEO_AGGREGATOR (mix); - gboolean same_downstream_gl_context = FALSE; - - if (!gst_gl_ensure_element_data (mix, &mix->display, &mix->other_context)) - return FALSE; - - gst_gl_display_filter_gl_api (mix->display, mixer_class->supported_gl_api); - - if (gst_query_find_allocation_meta (query, - GST_VIDEO_GL_TEXTURE_UPLOAD_META_API_TYPE, &idx)) { - GstGLContext *context; - const GstStructure *upload_meta_params; - gpointer handle; - gchar *type; - gchar *apis; - - gst_query_parse_nth_allocation_meta (query, idx, &upload_meta_params); - if (upload_meta_params) { - if (gst_structure_get (upload_meta_params, "gst.gl.GstGLContext", - GST_GL_TYPE_CONTEXT, &context, NULL) && context) { - GstGLContext *old = mix->context; - - mix->context = context; - if (old) - gst_object_unref (old); - same_downstream_gl_context = TRUE; - } else if (gst_structure_get (upload_meta_params, "gst.gl.context.handle", - G_TYPE_POINTER, &handle, "gst.gl.context.type", G_TYPE_STRING, - &type, "gst.gl.context.apis", G_TYPE_STRING, &apis, NULL) - && handle) { - GstGLPlatform platform; - GstGLAPI gl_apis; - - GST_DEBUG ("got GL context handle 0x%p with type %s and apis %s", - handle, type, apis); - - platform = gst_gl_platform_from_string (type); - gl_apis = gst_gl_api_from_string (apis); - - if (gl_apis && platform) - other_context = - gst_gl_context_new_wrapped (mix->display, (guintptr) handle, - platform, gl_apis); - } - } - } - - if (mix->other_context) { - if (!other_context) { - other_context = mix->other_context; - } else { - GST_ELEMENT_WARNING (mix, LIBRARY, SETTINGS, - ("%s", "Cannot share with more than one GL context"), - ("%s", "Cannot share with more than one GL context")); - } - } - - if (!mix->context) { - mix->context = gst_gl_context_new (mix->display); - if (!gst_gl_context_create (mix->context, other_context, &error)) - goto context_error; - } out_width = GST_VIDEO_INFO_WIDTH (&vagg->info); out_height = GST_VIDEO_INFO_HEIGHT (&vagg->info); @@ -820,12 +619,12 @@ gst_gl_mixer_decide_allocation (GstGLMixer * mix, GstQuery * query) g_mutex_lock (&mix->priv->gl_resource_lock); mix->priv->gl_resource_ready = FALSE; if (mix->fbo) { - gst_gl_context_del_fbo (mix->context, mix->fbo, mix->depthbuffer); + gst_gl_context_del_fbo (context, mix->fbo, mix->depthbuffer); mix->fbo = 0; mix->depthbuffer = 0; } - if (!gst_gl_context_gen_fbo (mix->context, out_width, out_height, + if (!gst_gl_context_gen_fbo (context, out_width, out_height, &mix->fbo, &mix->depthbuffer)) { g_cond_signal (&mix->priv->gl_resource_cond); g_mutex_unlock (&mix->priv->gl_resource_lock); @@ -833,12 +632,13 @@ gst_gl_mixer_decide_allocation (GstGLMixer * mix, GstQuery * query) } if (mix->out_tex_id) - gst_gl_context_del_texture (mix->context, &mix->out_tex_id); - gst_gl_context_gen_texture (mix->context, &mix->out_tex_id, + gst_gl_context_del_texture (context, &mix->out_tex_id); + gst_gl_context_gen_texture (context, &mix->out_tex_id, GST_VIDEO_FORMAT_RGBA, out_width, out_height); gst_query_parse_allocation (query, &caps, NULL); + mix->context = context; if (mixer_class->set_caps) mixer_class->set_caps (mix, caps); @@ -860,16 +660,8 @@ gst_gl_mixer_decide_allocation (GstGLMixer * mix, GstQuery * query) update_pool = FALSE; } - if (!pool || (!same_downstream_gl_context - && gst_query_find_allocation_meta (query, GST_GL_SYNC_META_API_TYPE, - NULL) - && !gst_buffer_pool_has_option (pool, - GST_BUFFER_POOL_OPTION_GL_SYNC_META))) { - /* can't use this pool */ - if (pool) - gst_object_unref (pool); - pool = gst_gl_buffer_pool_new (mix->context); - } + if (!pool) + pool = gst_gl_buffer_pool_new (context); config = gst_buffer_pool_get_config (pool); gst_buffer_pool_config_set_params (config, caps, size, min, max); @@ -899,104 +691,6 @@ context_error: } } -/* takes ownership of the pool, allocator and query */ -static gboolean -gst_gl_mixer_set_allocation (GstGLMixer * mix, - GstBufferPool * pool, GstAllocator * allocator, - GstAllocationParams * params, GstQuery * query) -{ - GstAllocator *oldalloc; - GstBufferPool *oldpool; - GstQuery *oldquery; - GstGLMixerPrivate *priv = mix->priv; - - GST_DEBUG ("storing allocation query"); - - GST_OBJECT_LOCK (mix); - oldpool = priv->pool; - priv->pool = pool; - priv->pool_active = FALSE; - - oldalloc = priv->allocator; - priv->allocator = allocator; - - oldquery = priv->query; - priv->query = query; - - if (params) - priv->params = *params; - else - gst_allocation_params_init (&priv->params); - GST_OBJECT_UNLOCK (mix); - - if (oldpool) { - GST_DEBUG_OBJECT (mix, "deactivating old pool %p", oldpool); - gst_buffer_pool_set_active (oldpool, FALSE); - gst_object_unref (oldpool); - } - if (oldalloc) { - gst_object_unref (oldalloc); - } - if (oldquery) { - gst_query_unref (oldquery); - } - return TRUE; -} - -static gboolean -gst_gl_mixer_do_bufferpool (GstGLMixer * mix, GstCaps * outcaps) -{ - GstQuery *query; - gboolean result = TRUE; - GstBufferPool *pool = NULL; - GstAllocator *allocator; - GstAllocationParams params; - GstAggregator *agg = GST_AGGREGATOR (mix); - - /* find a pool for the negotiated caps now */ - GST_DEBUG_OBJECT (mix, "doing allocation query"); - query = gst_query_new_allocation (outcaps, TRUE); - if (!gst_pad_peer_query (agg->srcpad, query)) { - /* not a problem, just debug a little */ - GST_DEBUG_OBJECT (mix, "peer ALLOCATION query failed"); - } - - GST_DEBUG_OBJECT (mix, "calling decide_allocation"); - result = gst_gl_mixer_decide_allocation (mix, query); - - GST_DEBUG_OBJECT (mix, "ALLOCATION (%d) params: %" GST_PTR_FORMAT, result, - query); - - if (!result) - goto no_decide_allocation; - - /* we got configuration from our peer or the decide_allocation method, - * parse them */ - if (gst_query_get_n_allocation_params (query) > 0) { - gst_query_parse_nth_allocation_param (query, 0, &allocator, ¶ms); - } else { - allocator = NULL; - gst_allocation_params_init (¶ms); - } - - if (gst_query_get_n_allocation_pools (query) > 0) - gst_query_parse_nth_allocation_pool (query, 0, &pool, NULL, NULL, NULL); - - /* now store */ - result = gst_gl_mixer_set_allocation (mix, pool, allocator, ¶ms, query); - - return result; - - /* Errors */ -no_decide_allocation: - { - GST_WARNING_OBJECT (mix, "Failed to decide allocation"); - gst_query_unref (query); - - return result; - } -} - static GstBuffer * _default_pad_upload_buffer (GstGLMixer * mix, GstGLMixerFrameData * frame, GstBuffer * buffer) @@ -1056,6 +750,7 @@ gst_gl_mixer_process_textures (GstGLMixer * mix, GstBuffer * outbuf) GstElement *element = GST_ELEMENT (mix); GstVideoAggregator *vagg = GST_VIDEO_AGGREGATOR (mix); GstGLMixerClass *mix_class = GST_GL_MIXER_GET_CLASS (mix); + GstGLContext *context = GST_GL_BASE_MIXER (mix)->context; GstGLMixerPrivate *priv = mix->priv; gboolean to_download = gst_caps_features_is_equal (GST_CAPS_FEATURES_MEMORY_SYSTEM_MEMORY, @@ -1082,7 +777,7 @@ gst_gl_mixer_process_textures (GstGLMixer * mix, GstBuffer * outbuf) "attempting to wrap for download"); if (!mix->download) - mix->download = gst_gl_download_new (mix->context); + mix->download = gst_gl_download_new (context); gst_gl_download_set_format (mix->download, &out_frame.info); out_tex = mix->out_tex_id; @@ -1198,14 +893,13 @@ gst_gl_mixer_process_buffers (GstGLMixer * mix, GstBuffer * outbuf) return mix_class->process_buffers (mix, mix->array_buffers, outbuf); } - - static GstFlowReturn gst_gl_mixer_aggregate_frames (GstVideoAggregator * vagg, GstBuffer * outbuf) { gboolean res = FALSE; GstGLMixer *mix = GST_GL_MIXER (vagg); GstGLMixerClass *mix_class = GST_GL_MIXER_GET_CLASS (vagg); + GstGLContext *context = GST_GL_BASE_MIXER (mix)->context; GstGLSyncMeta *sync_meta; if (mix_class->process_buffers) @@ -1215,7 +909,7 @@ gst_gl_mixer_aggregate_frames (GstVideoAggregator * vagg, GstBuffer * outbuf) sync_meta = gst_buffer_get_gl_sync_meta (outbuf); if (sync_meta) - gst_gl_sync_meta_set_sync_point (sync_meta, mix->context); + gst_gl_sync_meta_set_sync_point (sync_meta, context); return res ? GST_FLOW_OK : GST_FLOW_ERROR; } @@ -1224,12 +918,7 @@ static void gst_gl_mixer_get_property (GObject * object, guint prop_id, GValue * value, GParamSpec * pspec) { - GstGLMixer *mixer = GST_GL_MIXER (object); - switch (prop_id) { - case PROP_CONTEXT: - g_value_set_object (value, mixer->context); - break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); break; @@ -1296,7 +985,7 @@ gst_gl_mixer_start (GstAggregator * agg) GST_OBJECT_UNLOCK (mix); - return TRUE; + return GST_AGGREGATOR_CLASS (parent_class)->start (agg); } static gboolean @@ -1304,6 +993,7 @@ gst_gl_mixer_stop (GstAggregator * agg) { GstGLMixer *mix = GST_GL_MIXER (agg); GstGLMixerClass *mixer_class = GST_GL_MIXER_GET_CLASS (mix); + GstGLContext *context = GST_GL_BASE_MIXER (mix)->context; GST_OBJECT_LOCK (agg); g_ptr_array_free (mix->frames, TRUE); @@ -1315,7 +1005,7 @@ gst_gl_mixer_stop (GstAggregator * agg) if (mixer_class->reset) mixer_class->reset (mix); if (mix->fbo) { - gst_gl_context_del_fbo (mix->context, mix->fbo, mix->depthbuffer); + gst_gl_context_del_fbo (context, mix->fbo, mix->depthbuffer); mix->fbo = 0; mix->depthbuffer = 0; } @@ -1326,27 +1016,7 @@ gst_gl_mixer_stop (GstAggregator * agg) gst_aggregator_iterate_sinkpads (GST_AGGREGATOR (mix), _clean_upload, NULL); - if (mix->priv->query) { - gst_query_unref (mix->priv->query); - mix->priv->query = NULL; - } - - if (mix->priv->pool) { - gst_object_unref (mix->priv->pool); - mix->priv->pool = NULL; - } - - if (mix->display) { - gst_object_unref (mix->display); - mix->display = NULL; - } - - if (mix->context) { - gst_object_unref (mix->context); - mix->context = NULL; - } - gst_gl_mixer_reset (mix); - return TRUE; + return GST_AGGREGATOR_CLASS (parent_class)->stop (agg); } diff --git a/ext/gl/gstglmixer.h b/ext/gl/gstglmixer.h index 4a92d54713..a1fbb72286 100644 --- a/ext/gl/gstglmixer.h +++ b/ext/gl/gstglmixer.h @@ -24,10 +24,50 @@ #include #include #include -#include "gstglmixerpad.h" +#include "gstglbasemixer.h" G_BEGIN_DECLS +typedef struct _GstGLMixer GstGLMixer; +typedef struct _GstGLMixerClass GstGLMixerClass; +typedef struct _GstGLMixerPrivate GstGLMixerPrivate; +typedef struct _GstGLMixerFrameData GstGLMixerFrameData; + +#define GST_TYPE_GL_MIXER_PAD (gst_gl_mixer_pad_get_type()) +#define GST_GL_MIXER_PAD(obj) \ + (G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_GL_MIXER_PAD, GstGLMixerPad)) +#define GST_GL_MIXER_PAD_CLASS(klass) \ + (G_TYPE_CHECK_CLASS_CAST((klass),GST_TYPE_GL_MIXER_PAD, GstGLMixerPadClass)) +#define GST_IS_GL_MIXER_PAD(obj) \ + (G_TYPE_CHECK_INSTANCE_TYPE((obj),GST_TYPE_GL_MIXER_PAD)) +#define GST_IS_GL_MIXER_PAD_CLASS(klass) \ + (G_TYPE_CHECK_CLASS_TYPE((klass),GST_TYPE_GL_MIXER_PAD)) +#define GST_GL_MIXER_PAD_GET_CLASS(obj) \ + (G_TYPE_INSTANCE_GET_CLASS((obj),GST_TYPE_GL_MIXER_PAD,GstGLMixerPadClass)) + +typedef struct _GstGLMixerPad GstGLMixerPad; +typedef struct _GstGLMixerPadClass GstGLMixerPadClass; + +/* all information needed for one video stream */ +struct _GstGLMixerPad +{ + GstGLBaseMixerPad parent; /* subclass the pad */ + + /* */ + GstGLUpload *upload; + GstGLColorConvert *convert; + GstBuffer *gl_buffer; +}; + +struct _GstGLMixerPadClass +{ + GstGLBaseMixerPadClass parent_class; + + GstBuffer * (*upload_buffer) (GstGLMixer * mix, GstGLMixerFrameData * frame, GstBuffer * buffer); +}; + +GType gst_gl_mixer_pad_get_type (void); + #define GST_TYPE_GL_MIXER (gst_gl_mixer_get_type()) #define GST_GL_MIXER(obj) \ (G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_GL_MIXER, GstGLMixer)) @@ -40,11 +80,6 @@ G_BEGIN_DECLS #define GST_GL_MIXER_GET_CLASS(obj) \ (G_TYPE_INSTANCE_GET_CLASS((obj),GST_TYPE_GL_MIXER,GstGLMixerClass)) -typedef struct _GstGLMixer GstGLMixer; -typedef struct _GstGLMixerClass GstGLMixerClass; -typedef struct _GstGLMixerPrivate GstGLMixerPrivate; -typedef struct _GstGLMixerFrameData GstGLMixerFrameData; - typedef gboolean (*GstGLMixerSetCaps) (GstGLMixer* mixer, GstCaps* outcaps); typedef void (*GstGLMixerReset) (GstGLMixer *mixer); @@ -55,9 +90,10 @@ typedef gboolean (*GstGLMixerProcessTextures) (GstGLMixer *mix, struct _GstGLMixer { - GstVideoAggregator vaggregator; + GstGLBaseMixer vaggregator; - GstGLMixerPrivate *priv; + /* FIXME remove */ + GstGLContext *context; GPtrArray *array_buffers; GPtrArray *frames; @@ -65,17 +101,17 @@ struct _GstGLMixer GLuint out_tex_id; GstGLDownload *download; - GstGLDisplay *display; - GstGLContext *context, *other_context; GLuint fbo; GLuint depthbuffer; GstCaps *out_caps; + + GstGLMixerPrivate *priv; }; struct _GstGLMixerClass { - GstVideoAggregatorClass parent_class; + GstGLBaseMixerClass parent_class; GstGLAPI supported_gl_api; GstGLMixerSetCaps set_caps; diff --git a/ext/gl/gstglvideomixer.h b/ext/gl/gstglvideomixer.h index bc1cc7e7b8..b351888fce 100644 --- a/ext/gl/gstglvideomixer.h +++ b/ext/gl/gstglvideomixer.h @@ -22,7 +22,6 @@ #define _GST_GL_VIDEO_MIXER_H_ #include "gstglmixer.h" -#include "gstglmixerpad.h" G_BEGIN_DECLS From bf7409b8314d47a64486b727498509496cbb5e8d Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Wed, 25 Feb 2015 23:48:56 +1100 Subject: [PATCH 137/381] gl: new glmixerbin element --- ext/gl/gstglmixerbin.c | 570 +++++++++++++++++++++++++++++++++++++++++ ext/gl/gstglmixerbin.h | 72 ++++++ 2 files changed, 642 insertions(+) create mode 100644 ext/gl/gstglmixerbin.c create mode 100644 ext/gl/gstglmixerbin.h diff --git a/ext/gl/gstglmixerbin.c b/ext/gl/gstglmixerbin.c new file mode 100644 index 0000000000..2dada971d7 --- /dev/null +++ b/ext/gl/gstglmixerbin.c @@ -0,0 +1,570 @@ +/* + * + * GStreamer + * Copyright (C) 2015 Matthew Waters + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include + +#include "gstglmixerbin.h" + +#define GST_CAT_DEFAULT gst_gl_mixer_bin_debug +GST_DEBUG_CATEGORY (gst_gl_mixer_bin_debug); + +#define DEFAULT_LATENCY 0 + +struct input_chain +{ + GstGLMixerBin *self; + GstGhostPad *ghost_pad; + GstElement *upload; + GstElement *in_convert; + GstPad *mixer_pad; +}; + +static void +_free_input_chain (struct input_chain *chain) +{ + if (!chain) + return; + + if (chain->ghost_pad) + gst_object_unref (chain->ghost_pad); + chain->ghost_pad = NULL; + + if (chain->upload) { + gst_bin_remove (GST_BIN (chain->self), chain->upload); + gst_object_unref (chain->upload); + chain->upload = NULL; + } + + if (chain->in_convert) { + gst_bin_remove (GST_BIN (chain->self), chain->in_convert); + gst_object_unref (chain->in_convert); + chain->in_convert = NULL; + } + + if (chain->mixer_pad) { + gst_element_release_request_pad (chain->self->mixer, chain->mixer_pad); + gst_object_unref (chain->mixer_pad); + chain->mixer_pad = NULL; + } + + g_free (chain); +} + +struct _GstGLMixerBinPrivate +{ + gboolean running; + + GList *input_chains; +}; + +enum +{ + PROP_0, + PROP_MIXER, + PROP_LATENCY, +}; + +enum +{ + SIGNAL_0, + SIGNAL_CREATE_ELEMENT, + LAST_SIGNAL +}; + +static void gst_gl_mixer_bin_child_proxy_init (gpointer g_iface, + gpointer iface_data); + +#define GST_GL_MIXER_BIN_GET_PRIVATE(o) \ + (G_TYPE_INSTANCE_GET_PRIVATE((o), GST_TYPE_GL_MIXER_BIN, GstGLMixerBinPrivate)) +G_DEFINE_TYPE_WITH_CODE (GstGLMixerBin, gst_gl_mixer_bin, GST_TYPE_BIN, + G_IMPLEMENT_INTERFACE (GST_TYPE_CHILD_PROXY, + gst_gl_mixer_bin_child_proxy_init)); + +static guint gst_gl_mixer_bin_signals[LAST_SIGNAL] = { 0 }; + +static GstStaticPadTemplate src_factory = GST_STATIC_PAD_TEMPLATE ("src", + GST_PAD_SRC, + GST_PAD_ALWAYS, + GST_STATIC_CAPS ("video/x-raw(ANY)") + ); + +static GstStaticPadTemplate sink_factory = GST_STATIC_PAD_TEMPLATE ("sink_%u", + GST_PAD_SINK, + GST_PAD_REQUEST, + GST_STATIC_CAPS ("video/x-raw(ANY)") + ); + +static void gst_gl_mixer_bin_set_property (GObject * object, guint prop_id, + const GValue * value, GParamSpec * pspec); +static void gst_gl_mixer_bin_get_property (GObject * object, guint prop_id, + GValue * value, GParamSpec * pspec); +static void gst_gl_mixer_bin_dispose (GObject * object); + +static GstPad *gst_gl_mixer_bin_request_new_pad (GstElement * element, + GstPadTemplate * templ, const gchar * req_name, const GstCaps * caps); +static void gst_gl_mixer_bin_release_pad (GstElement * element, GstPad * pad); +static GstStateChangeReturn gst_gl_mixer_bin_change_state (GstElement * + element, GstStateChange transition); + +static void +gst_gl_mixer_bin_class_init (GstGLMixerBinClass * klass) +{ + GObjectClass *gobject_class = (GObjectClass *) klass; + GstElementClass *element_class = GST_ELEMENT_CLASS (klass); + + g_type_class_add_private (klass, sizeof (GstGLMixerBinPrivate)); + + GST_DEBUG_CATEGORY_INIT (GST_CAT_DEFAULT, "glmixerbin", 0, + "opengl mixer bin"); + + element_class->request_new_pad = gst_gl_mixer_bin_request_new_pad; + element_class->release_pad = gst_gl_mixer_bin_release_pad; + element_class->change_state = gst_gl_mixer_bin_change_state; + + gobject_class->get_property = gst_gl_mixer_bin_get_property; + gobject_class->set_property = gst_gl_mixer_bin_set_property; + gobject_class->dispose = GST_DEBUG_FUNCPTR (gst_gl_mixer_bin_dispose); + + g_object_class_install_property (gobject_class, PROP_MIXER, + g_param_spec_object ("mixer", + "GL mixer element", + "The GL mixer chain to use", + GST_TYPE_ELEMENT, + GST_PARAM_MUTABLE_READY | G_PARAM_READWRITE | + G_PARAM_STATIC_STRINGS)); + + g_object_class_install_property (gobject_class, PROP_LATENCY, + g_param_spec_int64 ("latency", "Buffer latency", + "Additional latency in live mode to allow upstream " + "to take longer to produce buffers for the current " + "position", 0, + (G_MAXLONG == G_MAXINT64) ? G_MAXINT64 : (G_MAXLONG * GST_SECOND - 1), + DEFAULT_LATENCY, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + + /** + * GstMixerBin::create-element: + * @object: the #GstGLMixerBin + * + * Will be emitted when we need the processing element/s that this bin will use + * + * Returns: a new #GstElement + */ + gst_gl_mixer_bin_signals[SIGNAL_CREATE_ELEMENT] = + g_signal_new ("create-element", G_TYPE_FROM_CLASS (klass), + G_SIGNAL_RUN_LAST, 0, NULL, NULL, g_cclosure_marshal_generic, + GST_TYPE_ELEMENT, 0); + + gst_element_class_add_pad_template (element_class, + gst_static_pad_template_get (&src_factory)); + gst_element_class_add_pad_template (element_class, + gst_static_pad_template_get (&sink_factory)); +} + +static void +gst_gl_mixer_bin_init (GstGLMixerBin * self) +{ + gboolean res = TRUE; + GstPad *pad; + + self->priv = GST_GL_MIXER_BIN_GET_PRIVATE (self); + + self->out_convert = gst_element_factory_make ("glcolorconvert", NULL); + self->download = gst_element_factory_make ("gldownload", NULL); + res &= gst_bin_add (GST_BIN (self), self->out_convert); + res &= gst_bin_add (GST_BIN (self), self->download); + + res &= + gst_element_link_pads (self->out_convert, "src", self->download, "sink"); + + pad = gst_element_get_static_pad (self->download, "src"); + if (!pad) { + res = FALSE; + } else { + GST_DEBUG_OBJECT (self, "setting target src pad %" GST_PTR_FORMAT, pad); + self->srcpad = gst_ghost_pad_new ("src", pad); + gst_element_add_pad (GST_ELEMENT_CAST (self), self->srcpad); + gst_object_unref (pad); + } + + if (!res) + GST_ERROR_OBJECT (self, "failed to create output chain"); +} + +static void +gst_gl_mixer_bin_dispose (GObject * object) +{ + GstGLMixerBin *self = GST_GL_MIXER_BIN (object); + GList *l = self->priv->input_chains; + + while (l) { + struct input_chain *chain = l->data; + + if (self->mixer && chain->mixer_pad) { + gst_element_release_request_pad (GST_ELEMENT (self->mixer), + chain->mixer_pad); + gst_object_unref (chain->mixer_pad); + chain->mixer_pad = NULL; + } + + l = l->next; + } + + g_list_free_full (self->priv->input_chains, (GDestroyNotify) g_free); + + G_OBJECT_CLASS (gst_gl_mixer_bin_parent_class)->dispose (object); +} + +static gboolean +_create_input_chain (GstGLMixerBin * self, struct input_chain *chain, + GstPad * mixer_pad) +{ + GstGLMixerBinClass *klass = GST_GL_MIXER_BIN_GET_CLASS (self); + GstPad *pad; + gboolean res = TRUE; + gchar *name; + + chain->self = self; + chain->mixer_pad = mixer_pad; + + chain->upload = gst_element_factory_make ("glupload", NULL); + chain->in_convert = gst_element_factory_make ("glcolorconvert", NULL); + + res &= gst_bin_add (GST_BIN (self), chain->in_convert); + res &= gst_bin_add (GST_BIN (self), chain->upload); + + pad = gst_element_get_static_pad (chain->in_convert, "src"); + if (gst_pad_link (pad, mixer_pad) != GST_PAD_LINK_OK) { + gst_object_unref (pad); + return FALSE; + } + gst_object_unref (pad); + res &= + gst_element_link_pads (chain->upload, "src", chain->in_convert, "sink"); + + pad = gst_element_get_static_pad (chain->upload, "sink"); + if (!pad) { + return FALSE; + } else { + GST_DEBUG_OBJECT (self, "setting target sink pad %" GST_PTR_FORMAT, pad); + name = gst_object_get_name (GST_OBJECT (mixer_pad)); + if (klass->create_input_pad) { + chain->ghost_pad = klass->create_input_pad (self, chain->mixer_pad); + gst_object_set_name (GST_OBJECT (chain->ghost_pad), name); + gst_ghost_pad_set_target (chain->ghost_pad, pad); + } else { + chain->ghost_pad = + GST_GHOST_PAD (gst_ghost_pad_new (GST_PAD_NAME (chain->mixer_pad), + pad)); + } + g_free (name); + + GST_OBJECT_LOCK (self); + if (self->priv->running) + gst_pad_set_active (GST_PAD (chain->ghost_pad), TRUE); + GST_OBJECT_UNLOCK (self); + + gst_element_add_pad (GST_ELEMENT_CAST (self), GST_PAD (chain->ghost_pad)); + gst_object_unref (pad); + } + + gst_element_sync_state_with_parent (chain->upload); + gst_element_sync_state_with_parent (chain->in_convert); + + return TRUE; +} + +static GstPadTemplate * +_find_element_pad_template (GstElement * element, + GstPadDirection direction, GstPadPresence presence) +{ + GstElementClass *klass = GST_ELEMENT_GET_CLASS (element); + GList *templ_list = gst_element_class_get_pad_template_list (klass); + GstPadTemplate *templ; + + /* find suitable template */ + while (templ_list) { + templ = (GstPadTemplate *) templ_list->data; + + if (GST_PAD_TEMPLATE_DIRECTION (templ) != direction + || GST_PAD_TEMPLATE_PRESENCE (templ) != presence) { + templ_list = templ_list->next; + templ = NULL; + continue; + } + + break; + } + + return templ; +} + +static gboolean +_connect_mixer_element (GstGLMixerBin * self) +{ + gboolean res = TRUE; + + g_return_val_if_fail (self->priv->input_chains == NULL, FALSE); + + gst_object_set_name (GST_OBJECT (self->mixer), "mixer"); + res &= gst_bin_add (GST_BIN (self), self->mixer); + + res &= gst_element_link_pads (self->mixer, "src", self->out_convert, "sink"); + + if (!res) + GST_ERROR_OBJECT (self, "Failed to link mixer element into the pipeline"); + + gst_element_sync_state_with_parent (self->mixer); + + return res; +} + +void +gst_gl_mixer_bin_finish_init_with_element (GstGLMixerBin * self, + GstElement * element) +{ + g_return_if_fail (GST_IS_ELEMENT (element)); + + self->mixer = element; + + if (!_connect_mixer_element (self)) { + gst_object_unref (self->mixer); + self->mixer = NULL; + } +} + +void +gst_gl_mixer_bin_finish_init (GstGLMixerBin * self) +{ + GstGLMixerBinClass *klass = GST_GL_MIXER_BIN_GET_CLASS (self); + GstElement *element = NULL; + + if (klass->create_element) + element = klass->create_element (); + + if (element) + gst_gl_mixer_bin_finish_init_with_element (self, element); +} + +static void +gst_gl_mixer_bin_get_property (GObject * object, + guint prop_id, GValue * value, GParamSpec * pspec) +{ + GstGLMixerBin *self = GST_GL_MIXER_BIN (object); + + switch (prop_id) { + case PROP_MIXER: + g_value_set_object (value, self->mixer); + break; + default: + if (self->mixer) + g_object_get_property (G_OBJECT (self->mixer), pspec->name, value); + break; + } +} + +static void +gst_gl_mixer_bin_set_property (GObject * object, + guint prop_id, const GValue * value, GParamSpec * pspec) +{ + GstGLMixerBin *self = GST_GL_MIXER_BIN (object); + + switch (prop_id) { + case PROP_MIXER: + { + GstElement *mixer = g_value_get_object (value); + /* FIXME: deal with replacing a mixer */ + g_return_if_fail (!self->mixer || (self->mixer == mixer)); + self->mixer = mixer; + if (mixer) + _connect_mixer_element (self); + break; + } + default: + if (self->mixer) + g_object_set_property (G_OBJECT (self->mixer), pspec->name, value); + break; + } +} + +static GstPad * +gst_gl_mixer_bin_request_new_pad (GstElement * element, GstPadTemplate * templ, + const gchar * req_name, const GstCaps * caps) +{ + GstGLMixerBin *self = GST_GL_MIXER_BIN (element); + GstPadTemplate *mixer_templ; + struct input_chain *chain; + GstPad *mixer_pad; + + chain = g_new0 (struct input_chain, 1); + + mixer_templ = _find_element_pad_template (self->mixer, + GST_PAD_TEMPLATE_DIRECTION (templ), GST_PAD_TEMPLATE_PRESENCE (templ)); + g_return_val_if_fail (mixer_templ, NULL); + + mixer_pad = + gst_element_request_pad (self->mixer, mixer_templ, req_name, NULL); + gst_object_unref (mixer_templ); + g_return_val_if_fail (mixer_pad, NULL); + + if (!_create_input_chain (self, chain, mixer_pad)) { + gst_element_release_request_pad (self->mixer, mixer_pad); + _free_input_chain (chain); + return NULL; + } + + GST_OBJECT_LOCK (element); + self->priv->input_chains = g_list_prepend (self->priv->input_chains, chain); + GST_OBJECT_UNLOCK (element); + + gst_child_proxy_child_added (GST_CHILD_PROXY (self), + G_OBJECT (chain->ghost_pad), GST_OBJECT_NAME (chain->ghost_pad)); + + return GST_PAD (chain->ghost_pad); +} + +static void +gst_gl_mixer_bin_release_pad (GstElement * element, GstPad * pad) +{ + GstGLMixerBin *self = GST_GL_MIXER_BIN (element); + GList *l = self->priv->input_chains; + + GST_OBJECT_LOCK (element); + while (l) { + struct input_chain *chain = l->data; + if (chain->mixer_pad == pad) { + _free_input_chain (chain); + self->priv->input_chains = + g_list_remove_link (self->priv->input_chains, l); + break; + } + l = l->next; + } + GST_OBJECT_UNLOCK (element); + + gst_element_remove_pad (element, pad); +} + +static GstStateChangeReturn +gst_gl_mixer_bin_change_state (GstElement * element, GstStateChange transition) +{ + GstGLMixerBin *self = GST_GL_MIXER_BIN (element); + GstGLMixerBinClass *klass = GST_GL_MIXER_BIN_GET_CLASS (self); + GstStateChangeReturn ret; + + switch (transition) { + case GST_STATE_CHANGE_NULL_TO_READY: + GST_OBJECT_LOCK (element); + if (!self->mixer) { + if (klass->create_element) + self->mixer = klass->create_element (); + + if (!self->mixer) + g_signal_emit (element, + gst_gl_mixer_bin_signals[SIGNAL_CREATE_ELEMENT], 0, &self->mixer); + + if (!self->mixer) { + GST_ERROR_OBJECT (element, "Failed to retreive element"); + GST_OBJECT_UNLOCK (element); + return GST_STATE_CHANGE_FAILURE; + } + GST_OBJECT_UNLOCK (element); + if (!_connect_mixer_element (self)) + return GST_STATE_CHANGE_FAILURE; + + GST_OBJECT_LOCK (element); + } + self->priv->running = TRUE; + GST_OBJECT_UNLOCK (element); + break; + default: + break; + } + + ret = + GST_ELEMENT_CLASS (gst_gl_mixer_bin_parent_class)->change_state (element, + transition); + if (ret == GST_STATE_CHANGE_FAILURE) + return ret; + + switch (transition) { + case GST_STATE_CHANGE_READY_TO_NULL: + GST_OBJECT_LOCK (self); + self->priv->running = FALSE; + GST_OBJECT_UNLOCK (self); + default: + break; + } + + return ret; +} + +static GObject * +gst_gl_mixer_bin_child_proxy_get_child_by_index (GstChildProxy * child_proxy, + guint index) +{ + GstGLMixerBin *mixer = GST_GL_MIXER_BIN (child_proxy); + GstBin *bin = GST_BIN_CAST (child_proxy); + GObject *res = NULL; + + GST_OBJECT_LOCK (bin); + /* XXX: not exactly thread safe with ordering */ + if (index < bin->numchildren) { + if ((res = g_list_nth_data (bin->children, index))) + gst_object_ref (res); + } else { + struct input_chain *chain; + if ((chain = + g_list_nth_data (mixer->priv->input_chains, + index - bin->numchildren))) { + res = gst_object_ref (chain->ghost_pad); + } + } + GST_OBJECT_UNLOCK (bin); + + return res; +} + +static guint +gst_gl_mixer_bin_child_proxy_get_children_count (GstChildProxy * child_proxy) +{ + GstGLMixerBin *mixer = GST_GL_MIXER_BIN (child_proxy); + GstBin *bin = GST_BIN_CAST (child_proxy); + guint num; + + GST_OBJECT_LOCK (bin); + num = bin->numchildren + g_list_length (mixer->priv->input_chains); + GST_OBJECT_UNLOCK (bin); + + return num; +} + +static void +gst_gl_mixer_bin_child_proxy_init (gpointer g_iface, gpointer iface_data) +{ + GstChildProxyInterface *iface = g_iface; + + iface->get_children_count = gst_gl_mixer_bin_child_proxy_get_children_count; + iface->get_child_by_index = gst_gl_mixer_bin_child_proxy_get_child_by_index; +} diff --git a/ext/gl/gstglmixerbin.h b/ext/gl/gstglmixerbin.h new file mode 100644 index 0000000000..5e5bb60b43 --- /dev/null +++ b/ext/gl/gstglmixerbin.h @@ -0,0 +1,72 @@ +/* + * GStreamer + * Copyright (C) 2009 Julien Isorce + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifndef __GST_GL_MIXER_BIN_H__ +#define __GST_GL_MIXER_BIN_H__ + +#include +#include +#include + +G_BEGIN_DECLS + +GType gst_gl_mixer_bin_get_type(void); +#define GST_TYPE_GL_MIXER_BIN (gst_gl_mixer_bin_get_type()) +#define GST_GL_MIXER_BIN(obj) \ + (G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_GL_MIXER_BIN, GstGLMixerBin)) +#define GST_GL_MIXER_BIN_CLASS(klass) \ + (G_TYPE_CHECK_CLASS_CAST((klass),GST_TYPE_GL_MIXER_BIN, GstGLMixerBinClass)) +#define GST_IS_GL_MIXER_BIN(obj) \ + (G_TYPE_CHECK_INSTANCE_TYPE((obj),GST_TYPE_GL_MIXER_BIN)) +#define GST_IS_GL_MIXER_BIN_CLASS(klass) \ + (G_TYPE_CHECK_CLASS_TYPE((klass),GST_TYPE_GL_MIXER_BIN)) +#define GST_GL_MIXER_BIN_GET_CLASS(obj) \ + (G_TYPE_INSTANCE_GET_CLASS((obj),GST_TYPE_GL_MIXER_BIN,GstGLMixerBinClass)) + +typedef struct _GstGLMixerBin GstGLMixerBin; +typedef struct _GstGLMixerBinClass GstGLMixerBinClass; +typedef struct _GstGLMixerBinPrivate GstGLMixerBinPrivate; + +struct _GstGLMixerBin +{ + GstBin parent; + + GstElement *mixer; + GstElement *out_convert; + GstElement *download; + GstPad *srcpad; + + GstGLMixerBinPrivate *priv; +}; + +struct _GstGLMixerBinClass +{ + GstBinClass parent_class; + + GstElement * (*create_element) (void); + GstGhostPad * (*create_input_pad) (GstGLMixerBin * self, GstPad * mixer_pad); +}; + +void gst_gl_mixer_bin_finish_init (GstGLMixerBin * self); +void gst_gl_mixer_bin_finish_init_with_element (GstGLMixerBin * self, + GstElement * element); + +G_END_DECLS +#endif /* __GST_GL_MIXER_BIN_H__ */ From b6cf2d9f21640880258060dea7af45d72243a3ed Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Thu, 26 Feb 2015 00:20:37 +1100 Subject: [PATCH 138/381] glmixer: remove usage of upload/download objects --- ext/gl/gstglmixer.c | 296 ++++++-------------------------------------- ext/gl/gstglmixer.h | 14 +-- 2 files changed, 40 insertions(+), 270 deletions(-) diff --git a/ext/gl/gstglmixer.c b/ext/gl/gstglmixer.c index 633a11e316..a1da5f4e63 100644 --- a/ext/gl/gstglmixer.c +++ b/ext/gl/gstglmixer.c @@ -43,8 +43,6 @@ static void gst_gl_mixer_pad_get_property (GObject * object, guint prop_id, static void gst_gl_mixer_pad_set_property (GObject * object, guint prop_id, const GValue * value, GParamSpec * pspec); static void gst_gl_mixer_pad_finalize (GObject * object); -static GstBuffer *_default_pad_upload_buffer (GstGLMixer * mix, - GstGLMixerFrameData * frame, GstBuffer * buffer); enum { @@ -80,64 +78,14 @@ gst_gl_mixer_pad_class_init (GstGLMixerPadClass * klass) vaggpad_class->set_info = NULL; vaggpad_class->prepare_frame = NULL; vaggpad_class->clean_frame = NULL; - - klass->upload_buffer = _default_pad_upload_buffer; } static void gst_gl_mixer_pad_finalize (GObject * object) { - GstGLMixerPad *pad = GST_GL_MIXER_PAD (object); - - if (pad->upload) { - gst_object_unref (pad->upload); - pad->upload = NULL; - } - G_OBJECT_CLASS (gst_gl_mixer_pad_parent_class)->finalize (object); } -static void -_init_upload (GstGLMixer * mix, GstGLMixerPad * pad) -{ - GstVideoAggregatorPad *vaggpad = GST_VIDEO_AGGREGATOR_PAD (pad); - GstGLContext *context = GST_GL_BASE_MIXER (mix)->context; - - if (!pad->upload) { - GstCaps *in_caps = gst_pad_get_current_caps (GST_PAD (pad)); - GstCaps *upload_caps = gst_caps_copy (in_caps); - GstCapsFeatures *gl_features = - gst_caps_features_from_string (GST_CAPS_FEATURE_MEMORY_GL_MEMORY); - - pad->upload = gst_gl_upload_new (context); - - gst_caps_set_features (upload_caps, 0, - gst_caps_features_copy (gl_features)); - gst_gl_upload_set_caps (pad->upload, in_caps, upload_caps); - gst_caps_unref (in_caps); - - if (!pad->convert) { - GstVideoInfo gl_info; - GstCaps *gl_caps; - - gst_video_info_set_format (&gl_info, - GST_VIDEO_FORMAT_RGBA, - GST_VIDEO_INFO_WIDTH (&vaggpad->info), - GST_VIDEO_INFO_HEIGHT (&vaggpad->info)); - gl_caps = gst_video_info_to_caps (&gl_info); - gst_caps_set_features (gl_caps, 0, gst_caps_features_copy (gl_features)); - - pad->convert = gst_gl_color_convert_new (context); - - gst_gl_color_convert_set_caps (pad->convert, upload_caps, gl_caps); - gst_caps_unref (gl_caps); - } - - gst_caps_unref (upload_caps); - gst_caps_features_free (gl_features); - } -} - static void gst_gl_mixer_pad_get_property (GObject * object, guint prop_id, GValue * value, GParamSpec * pspec) @@ -185,7 +133,6 @@ gst_gl_mixer_propose_allocation (GstGLBaseMixer * base_mix, GstGLBaseMixerPad * base_pad, GstQuery * decide_query, GstQuery * query) { GstGLMixer *mix = GST_GL_MIXER (base_mix); - GstGLMixerPad *pad = GST_GL_MIXER_PAD (base_pad); GstGLContext *context = base_mix->context; GstBufferPool *pool = NULL; GstStructure *config; @@ -225,10 +172,6 @@ gst_gl_mixer_propose_allocation (GstGLBaseMixer * base_mix, if (context->gl_vtable->FenceSync) gst_query_add_allocation_meta (query, GST_GL_SYNC_META_API_TYPE, 0); - _init_upload (mix, pad); - - gst_gl_upload_propose_allocation (pad->upload, decide_query, query); - return TRUE; /* ERRORS */ @@ -269,34 +212,12 @@ gst_gl_mixer_pad_sink_acceptcaps (GstPad * pad, GstGLMixer * mix, return ret; } +/* copies the given caps */ static GstCaps * -gst_gl_mixer_set_caps_features (const GstCaps * caps, - const gchar * feature_name) +_update_caps (GstVideoAggregator * vagg, GstCaps * caps) { - GstCaps *ret = gst_gl_caps_replace_all_caps_features (caps, feature_name); - gst_caps_set_simple (ret, "format", G_TYPE_STRING, "RGBA", NULL); - return ret; -} - -GstCaps * -gst_gl_mixer_update_caps (GstGLMixer * mix, GstCaps * caps) -{ - GstGLContext *context = GST_GL_BASE_MIXER (mix)->context; - GstCaps *result, *tmp, *gl_caps; - - gl_caps = gst_caps_from_string ("video/x-raw(memory:GLMemory),format=RGBA"); - - result = - gst_gl_color_convert_transform_caps (context, GST_PAD_SRC, gl_caps, NULL); - tmp = result; - GST_DEBUG_OBJECT (mix, "convert returned caps %" GST_PTR_FORMAT, tmp); - - result = gst_gl_upload_transform_caps (context, GST_PAD_SRC, tmp, NULL); - gst_caps_unref (tmp); - tmp = result; - GST_DEBUG_OBJECT (mix, "transfer returned caps %" GST_PTR_FORMAT, tmp); - - return result; + return gst_gl_caps_replace_all_caps_features (caps, + GST_CAPS_FEATURE_MEMORY_GL_MEMORY); } static GstCaps * @@ -315,8 +236,7 @@ gst_gl_mixer_pad_sink_getcaps (GstPad * pad, GstGLMixer * mix, GstCaps * filter) had_current_caps = FALSE; sinkcaps = template_caps; } else { - sinkcaps = - gst_caps_merge (sinkcaps, gst_gl_mixer_update_caps (mix, sinkcaps)); + sinkcaps = gst_caps_merge (sinkcaps, template_caps); } filtered_caps = sinkcaps; @@ -395,11 +315,7 @@ static GstStaticPadTemplate src_factory = GST_STATIC_PAD_TEMPLATE ("src", GST_PAD_ALWAYS, GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES (GST_CAPS_FEATURE_MEMORY_GL_MEMORY, - "RGBA") "; " - GST_VIDEO_CAPS_MAKE_WITH_FEATURES - (GST_CAPS_FEATURE_META_GST_VIDEO_GL_TEXTURE_UPLOAD_META, - "RGBA") - "; " GST_VIDEO_CAPS_MAKE (GST_GL_COLOR_CONVERT_FORMATS)) + "RGBA")) ); static GstStaticPadTemplate sink_factory = GST_STATIC_PAD_TEMPLATE ("sink_%u", @@ -407,15 +323,7 @@ static GstStaticPadTemplate sink_factory = GST_STATIC_PAD_TEMPLATE ("sink_%u", GST_PAD_REQUEST, GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES (GST_CAPS_FEATURE_MEMORY_GL_MEMORY, - "RGBA") "; " -#if GST_GL_HAVE_PLATFORM_EGL - GST_VIDEO_CAPS_MAKE_WITH_FEATURES (GST_CAPS_FEATURE_MEMORY_EGL_IMAGE, - "RGBA") "; " -#endif - GST_VIDEO_CAPS_MAKE_WITH_FEATURES - (GST_CAPS_FEATURE_META_GST_VIDEO_GL_TEXTURE_UPLOAD_META, - "RGBA") - "; " GST_VIDEO_CAPS_MAKE (GST_GL_COLOR_CONVERT_FORMATS)) + "RGBA")) ); static gboolean gst_gl_mixer_src_query (GstAggregator * agg, GstQuery * query); @@ -471,6 +379,7 @@ gst_gl_mixer_class_init (GstGLMixerClass * klass) videoaggregator_class->aggregate_frames = gst_gl_mixer_aggregate_frames; videoaggregator_class->get_output_buffer = gst_gl_mixer_get_output_buffer; videoaggregator_class->negotiated_caps = _negotiated_caps; + videoaggregator_class->update_caps = _update_caps; videoaggregator_class->find_best_format = NULL; mix_class->propose_allocation = gst_gl_mixer_propose_allocation; @@ -518,27 +427,19 @@ gst_gl_mixer_finalize (GObject * object) static gboolean gst_gl_mixer_query_caps (GstPad * pad, GstAggregator * agg, GstQuery * query) { - GstGLMixer *mix = GST_GL_MIXER (agg); - GstGLContext *context = GST_GL_BASE_MIXER (mix)->context; - GstCaps *filter, *current_caps, *retcaps, *gl_caps; + GstCaps *filter, *current_caps, *retcaps, *template_caps; gst_query_parse_caps (query, &filter); + template_caps = gst_pad_get_pad_template_caps (agg->srcpad); + current_caps = gst_pad_get_current_caps (pad); if (current_caps == NULL) - current_caps = gst_pad_get_pad_template_caps (agg->srcpad); - - /* convert from current caps to GLMemory caps */ - gl_caps = - gst_caps_merge (gst_gl_mixer_set_caps_features - (current_caps, GST_CAPS_FEATURE_MEMORY_GL_MEMORY), - gst_gl_mixer_set_caps_features (current_caps, - GST_CAPS_FEATURE_META_GST_VIDEO_GL_TEXTURE_UPLOAD_META)); - retcaps = - gst_gl_download_transform_caps (context, GST_PAD_SINK, current_caps, - NULL); - retcaps = gst_caps_merge (gl_caps, retcaps); - gst_caps_unref (current_caps); + retcaps = gst_caps_ref (template_caps); + else { + retcaps = gst_caps_merge (current_caps, template_caps); + template_caps = NULL; + } if (filter) { current_caps = @@ -550,6 +451,9 @@ gst_gl_mixer_query_caps (GstPad * pad, GstAggregator * agg, GstQuery * query) gst_query_set_caps_result (query, retcaps); gst_caps_unref (retcaps); + if (template_caps) + gst_caps_unref (template_caps); + return TRUE; } @@ -631,11 +535,6 @@ gst_gl_mixer_decide_allocation (GstGLBaseMixer * base_mix, GstQuery * query) goto context_error; } - if (mix->out_tex_id) - gst_gl_context_del_texture (context, &mix->out_tex_id); - gst_gl_context_gen_texture (context, &mix->out_tex_id, - GST_VIDEO_FORMAT_RGBA, out_width, out_height); - gst_query_parse_allocation (query, &caps, NULL); mix->context = context; @@ -666,11 +565,6 @@ gst_gl_mixer_decide_allocation (GstGLBaseMixer * base_mix, GstQuery * query) gst_buffer_pool_config_set_params (config, caps, size, min, max); gst_buffer_pool_config_add_option (config, GST_BUFFER_POOL_OPTION_VIDEO_META); - if (gst_query_find_allocation_meta (query, GST_GL_SYNC_META_API_TYPE, NULL)) - gst_buffer_pool_config_add_option (config, - GST_BUFFER_POOL_OPTION_GL_SYNC_META); - gst_buffer_pool_config_add_option (config, - GST_BUFFER_POOL_OPTION_VIDEO_GL_TEXTURE_UPLOAD_META); gst_buffer_pool_set_config (pool, config); @@ -691,98 +585,28 @@ context_error: } } -static GstBuffer * -_default_pad_upload_buffer (GstGLMixer * mix, GstGLMixerFrameData * frame, - GstBuffer * buffer) -{ - GstVideoAggregatorPad *vaggpad = GST_VIDEO_AGGREGATOR_PAD (frame->pad); - GstGLMixerPad *pad = frame->pad; - GstBuffer *uploaded_buf, *gl_buffer; - GstVideoInfo gl_info; - GstVideoFrame gl_frame; - GstGLSyncMeta *sync_meta; - - gst_video_info_set_format (&gl_info, - GST_VIDEO_FORMAT_RGBA, - GST_VIDEO_INFO_WIDTH (&vaggpad->info), - GST_VIDEO_INFO_HEIGHT (&vaggpad->info)); - - _init_upload (mix, pad); - - sync_meta = gst_buffer_get_gl_sync_meta (vaggpad->buffer); - if (sync_meta) - gst_gl_sync_meta_wait (sync_meta); - - if (gst_gl_upload_perform_with_buffer (pad->upload, - vaggpad->buffer, &uploaded_buf) != GST_GL_UPLOAD_DONE) { - return NULL; - } - - if (!(gl_buffer = gst_gl_color_convert_perform (pad->convert, uploaded_buf))) { - gst_buffer_unref (uploaded_buf); - return NULL; - } - - if (!gst_video_frame_map (&gl_frame, &gl_info, gl_buffer, - GST_MAP_READ | GST_MAP_GL)) { - gst_buffer_unref (uploaded_buf); - gst_buffer_unref (gl_buffer); - return NULL; - } - - frame->texture = *(guint *) gl_frame.data[0]; - - gst_buffer_unref (uploaded_buf); - gst_video_frame_unmap (&gl_frame); - - return gl_buffer; -} - gboolean gst_gl_mixer_process_textures (GstGLMixer * mix, GstBuffer * outbuf) { guint i; GList *walk; - guint out_tex, out_tex_target; + guint out_tex; gboolean res = TRUE; guint array_index = 0; GstVideoFrame out_frame; GstElement *element = GST_ELEMENT (mix); GstVideoAggregator *vagg = GST_VIDEO_AGGREGATOR (mix); GstGLMixerClass *mix_class = GST_GL_MIXER_GET_CLASS (mix); - GstGLContext *context = GST_GL_BASE_MIXER (mix)->context; GstGLMixerPrivate *priv = mix->priv; - gboolean to_download = - gst_caps_features_is_equal (GST_CAPS_FEATURES_MEMORY_SYSTEM_MEMORY, - gst_caps_get_features (mix->out_caps, 0)); - GstMapFlags out_map_flags = GST_MAP_WRITE; GST_TRACE ("Processing buffers"); - to_download |= !gst_is_gl_memory (gst_buffer_peek_memory (outbuf, 0)); - - if (!to_download) - out_map_flags |= GST_MAP_GL; - - if (!gst_video_frame_map (&out_frame, &vagg->info, outbuf, out_map_flags)) { + if (!gst_video_frame_map (&out_frame, &vagg->info, outbuf, + GST_MAP_WRITE | GST_MAP_GL)) { return FALSE; } - if (!to_download) { - out_tex = *(guint *) out_frame.data[0]; - out_tex_target = - ((GstGLMemory *) gst_buffer_peek_memory (outbuf, 0))->tex_target; - } else { - GST_INFO ("Output Buffer does not contain correct memory, " - "attempting to wrap for download"); - - if (!mix->download) - mix->download = gst_gl_download_new (context); - - gst_gl_download_set_format (mix->download, &out_frame.info); - out_tex = mix->out_tex_id; - out_tex_target = GL_TEXTURE_2D; - } + out_tex = *(guint *) out_frame.data[0]; GST_OBJECT_LOCK (mix); walk = element->sinkpads; @@ -793,7 +617,6 @@ gst_gl_mixer_process_textures (GstGLMixer * mix, GstBuffer * outbuf) mix->frames->pdata[i] = g_slice_new0 (GstGLMixerFrameData); while (walk) { GstGLMixerPad *pad = GST_GL_MIXER_PAD (walk->data); - GstGLMixerPadClass *pad_class = GST_GL_MIXER_PAD_GET_CLASS (pad); GstVideoAggregatorPad *vaggpad = walk->data; GstGLMixerFrameData *frame; @@ -804,15 +627,24 @@ gst_gl_mixer_process_textures (GstGLMixer * mix, GstBuffer * outbuf) walk = g_list_next (walk); if (vaggpad->buffer != NULL) { - g_assert (pad_class->upload_buffer); + GstVideoInfo gl_info; + GstVideoFrame gl_frame; + GstGLSyncMeta *sync_meta; - if (pad->gl_buffer) - gst_buffer_unref (pad->gl_buffer); - pad->gl_buffer = pad_class->upload_buffer (mix, frame, vaggpad->buffer); + gst_video_info_set_format (&gl_info, + GST_VIDEO_FORMAT_RGBA, + GST_VIDEO_INFO_WIDTH (&vaggpad->info), + GST_VIDEO_INFO_HEIGHT (&vaggpad->info)); - GST_DEBUG_OBJECT (pad, - "uploaded buffer %" GST_PTR_FORMAT " from buffer %" GST_PTR_FORMAT, - pad->gl_buffer, vaggpad->buffer); + sync_meta = gst_buffer_get_gl_sync_meta (vaggpad->buffer); + if (sync_meta) + gst_gl_sync_meta_wait (sync_meta); + + if (gst_video_frame_map (&gl_frame, &gl_info, vaggpad->buffer, + GST_MAP_READ | GST_MAP_GL)) { + frame->texture = *(guint *) gl_frame.data[0]; + gst_video_frame_unmap (&gl_frame); + } } ++array_index; @@ -834,28 +666,7 @@ gst_gl_mixer_process_textures (GstGLMixer * mix, GstBuffer * outbuf) g_mutex_unlock (&priv->gl_resource_lock); - if (to_download) { - if (!gst_gl_download_perform_with_data (mix->download, - out_tex, out_tex_target, out_frame.data)) { - GST_ELEMENT_ERROR (mix, RESOURCE, NOT_FOUND, ("%s", - "Failed to download video frame"), (NULL)); - res = FALSE; - goto out; - } - } - out: - i = 0; - walk = GST_ELEMENT (mix)->sinkpads; - while (walk) { - GstGLMixerPad *pad = GST_GL_MIXER_PAD (walk->data); - - if (pad->upload) - gst_gl_upload_release_buffer (pad->upload); - - walk = g_list_next (walk); - i++; - } GST_OBJECT_UNLOCK (mix); gst_video_frame_unmap (&out_frame); @@ -936,29 +747,6 @@ gst_gl_mixer_set_property (GObject * object, } } -static gboolean -_clean_upload (GstAggregator * agg, GstAggregatorPad * aggpad, gpointer udata) -{ - GstGLMixerPad *pad = GST_GL_MIXER_PAD (aggpad); - - if (pad->gl_buffer) { - gst_buffer_unref (pad->gl_buffer); - pad->gl_buffer = NULL; - } - - if (pad->upload) { - gst_object_unref (pad->upload); - pad->upload = NULL; - } - - if (pad->convert) { - gst_object_unref (pad->convert); - pad->convert = NULL; - } - - return TRUE; -} - static void _free_glmixer_frame_data (GstGLMixerFrameData * frame) { @@ -1009,12 +797,6 @@ gst_gl_mixer_stop (GstAggregator * agg) mix->fbo = 0; mix->depthbuffer = 0; } - if (mix->download) { - gst_object_unref (mix->download); - mix->download = NULL; - } - - gst_aggregator_iterate_sinkpads (GST_AGGREGATOR (mix), _clean_upload, NULL); gst_gl_mixer_reset (mix); diff --git a/ext/gl/gstglmixer.h b/ext/gl/gstglmixer.h index a1fbb72286..5b334750e4 100644 --- a/ext/gl/gstglmixer.h +++ b/ext/gl/gstglmixer.h @@ -51,19 +51,12 @@ typedef struct _GstGLMixerPadClass GstGLMixerPadClass; /* all information needed for one video stream */ struct _GstGLMixerPad { - GstGLBaseMixerPad parent; /* subclass the pad */ - - /* */ - GstGLUpload *upload; - GstGLColorConvert *convert; - GstBuffer *gl_buffer; + GstGLBaseMixerPad parent; }; struct _GstGLMixerPadClass { GstGLBaseMixerPadClass parent_class; - - GstBuffer * (*upload_buffer) (GstGLMixer * mix, GstGLMixerFrameData * frame, GstBuffer * buffer); }; GType gst_gl_mixer_pad_get_type (void); @@ -92,15 +85,11 @@ struct _GstGLMixer { GstGLBaseMixer vaggregator; - /* FIXME remove */ GstGLContext *context; GPtrArray *array_buffers; GPtrArray *frames; - GLuint out_tex_id; - GstGLDownload *download; - GLuint fbo; GLuint depthbuffer; @@ -129,7 +118,6 @@ struct _GstGLMixerFrameData GType gst_gl_mixer_get_type(void); gboolean gst_gl_mixer_process_textures (GstGLMixer * mix, GstBuffer * outbuf); -GstCaps * gst_gl_mixer_update_caps (GstGLMixer * mix, GstCaps * caps); G_END_DECLS #endif /* __GST_GL_MIXER_H__ */ From acd9a252c095f290ce585cf69e77231ccbc6dc7f Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Thu, 26 Feb 2015 13:45:56 +1100 Subject: [PATCH 139/381] glvideomixer: implement with glmixerbin The relevant properties are forwarded to/from the containing bin and sink pads. --- ext/gl/gstglvideomixer.c | 313 ++++++++++++++++++++++++++++++++------- ext/gl/gstglvideomixer.h | 1 + 2 files changed, 263 insertions(+), 51 deletions(-) diff --git a/ext/gl/gstglvideomixer.c b/ext/gl/gstglvideomixer.c index c8aebcbac7..ad73404740 100644 --- a/ext/gl/gstglvideomixer.c +++ b/ext/gl/gstglvideomixer.c @@ -44,16 +44,273 @@ #endif #include "gstglvideomixer.h" +#include "gstglmixerbin.h" #define GST_CAT_DEFAULT gst_gl_video_mixer_debug GST_DEBUG_CATEGORY_STATIC (GST_CAT_DEFAULT); +#define GST_GL_TYPE_VIDEO_MIXER_BACKGROUND (gst_gl_video_mixer_background_get_type()) +static GType +gst_gl_video_mixer_background_get_type (void) +{ + static GType mixer_background_type = 0; + + static const GEnumValue mixer_background[] = { + {GST_GL_VIDEO_MIXER_BACKGROUND_CHECKER, "Checker pattern", "checker"}, + {GST_GL_VIDEO_MIXER_BACKGROUND_BLACK, "Black", "black"}, + {GST_GL_VIDEO_MIXER_BACKGROUND_WHITE, "White", "white"}, + {GST_GL_VIDEO_MIXER_BACKGROUND_TRANSPARENT, + "Transparent Background to enable further compositing", "transparent"}, + {0, NULL, NULL}, + }; + + if (!mixer_background_type) { + mixer_background_type = + g_enum_register_static ("GstGLVideoMixerBackground", mixer_background); + } + return mixer_background_type; +} + +#define DEFAULT_PAD_XPOS 0 +#define DEFAULT_PAD_YPOS 0 +#define DEFAULT_PAD_WIDTH 0 +#define DEFAULT_PAD_HEIGHT 0 +#define DEFAULT_PAD_ALPHA 1.0 +#define DEFAULT_PAD_ZORDER 0 + +enum +{ + PROP_INPUT_0, + PROP_INPUT_XPOS, + PROP_INPUT_YPOS, + PROP_INPUT_WIDTH, + PROP_INPUT_HEIGHT, + PROP_INPUT_ALPHA, + PROP_INPUT_ZORDER, +}; + +static void gst_gl_video_mixer_input_get_property (GObject * object, + guint prop_id, GValue * value, GParamSpec * pspec); +static void gst_gl_video_mixer_input_set_property (GObject * object, + guint prop_id, const GValue * value, GParamSpec * pspec); + +static GstFlowReturn gst_gl_video_mixer_input_chain (GstPad * pad, + GstObject * parent, GstBuffer * buffer); +static GstFlowReturn gst_gl_video_mixer_input_event (GstPad * pad, + GstObject * parent, GstEvent * event); + +typedef struct _GstGLVideoMixerInput GstGLVideoMixerInput; +typedef GstGhostPadClass GstGLVideoMixerInputClass; + +struct _GstGLVideoMixerInput +{ + GstGhostPad parent; + + GstSegment segment; + + GstPad *mixer_pad; +}; + +GType gst_gl_video_mixer_input_get_type (void); +G_DEFINE_TYPE (GstGLVideoMixerInput, gst_gl_video_mixer_input, + GST_TYPE_GHOST_PAD); + +static void +gst_gl_video_mixer_input_init (GstGLVideoMixerInput * self) +{ + GstPad *pad = GST_PAD (self); + + gst_pad_set_event_function (pad, gst_gl_video_mixer_input_event); +} + +static void +gst_gl_video_mixer_input_class_init (GstGLVideoMixerInputClass * klass) +{ + GObjectClass *gobject_class = G_OBJECT_CLASS (klass); + + gobject_class->set_property = gst_gl_video_mixer_input_set_property; + gobject_class->get_property = gst_gl_video_mixer_input_get_property; + + g_object_class_install_property (gobject_class, PROP_INPUT_ZORDER, + g_param_spec_uint ("zorder", "Z-Order", "Z Order of the picture", + 0, 10000, DEFAULT_PAD_ZORDER, + G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); + g_object_class_install_property (gobject_class, PROP_INPUT_XPOS, + g_param_spec_int ("xpos", "X Position", "X Position of the picture", + G_MININT, G_MAXINT, DEFAULT_PAD_XPOS, + G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); + g_object_class_install_property (gobject_class, PROP_INPUT_YPOS, + g_param_spec_int ("ypos", "Y Position", "Y Position of the picture", + G_MININT, G_MAXINT, DEFAULT_PAD_YPOS, + G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); + g_object_class_install_property (gobject_class, PROP_INPUT_WIDTH, + g_param_spec_int ("width", "Width", "Width of the picture", + G_MININT, G_MAXINT, DEFAULT_PAD_WIDTH, + G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); + g_object_class_install_property (gobject_class, PROP_INPUT_HEIGHT, + g_param_spec_int ("height", "Height", "Height of the picture", + G_MININT, G_MAXINT, DEFAULT_PAD_HEIGHT, + G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); + g_object_class_install_property (gobject_class, PROP_INPUT_ALPHA, + g_param_spec_double ("alpha", "Alpha", "Alpha of the picture", 0.0, 1.0, + DEFAULT_PAD_ALPHA, + G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); +} + +static void +gst_gl_video_mixer_input_get_property (GObject * object, guint prop_id, + GValue * value, GParamSpec * pspec) +{ + GstGLVideoMixerInput *self = (GstGLVideoMixerInput *) object; + + if (self->mixer_pad) + g_object_get_property (G_OBJECT (self->mixer_pad), pspec->name, value); +} + +static void +gst_gl_video_mixer_input_set_property (GObject * object, guint prop_id, + const GValue * value, GParamSpec * pspec) +{ + GstGLVideoMixerInput *self = (GstGLVideoMixerInput *) object; + + if (self->mixer_pad) + g_object_set_property (G_OBJECT (self->mixer_pad), pspec->name, value); +} + +static GstFlowReturn +gst_gl_video_mixer_input_chain (GstPad * pad, GstObject * parent, + GstBuffer * buffer) +{ + GstGLVideoMixerInput *self = (GstGLVideoMixerInput *) pad; + GstClockTime timestamp, stream_time; +// gdouble alpha; + + timestamp = GST_BUFFER_TIMESTAMP (buffer); + + stream_time = + gst_segment_to_stream_time (&self->segment, GST_FORMAT_TIME, timestamp); + + gst_object_sync_values (GST_OBJECT (self), stream_time); +#if 0 + /* FIXME: implement no-upload on alpha = 0 */ + g_object_get (self, "alpha", &alpha, NULL); + + if (alpha <= 0.0) { + GST_DEBUG_OBJECT (self, "dropping buffer %" GST_PTR_FORMAT + " due to alpha value %f", buffer, alpha); + gst_buffer_unref (buffer); + return GST_FLOW_OK; + } +#endif + return gst_proxy_pad_chain_default (pad, parent, buffer); +} + +static GstFlowReturn +gst_gl_video_mixer_input_event (GstPad * pad, GstObject * parent, + GstEvent * event) +{ + GstGLVideoMixerInput *self = (GstGLVideoMixerInput *) pad; + + switch (GST_EVENT_TYPE (event)) { + case GST_EVENT_SEGMENT: + gst_event_copy_segment (event, &self->segment); + break; + default: + break; + } + + return gst_pad_event_default (pad, parent, event); +} + +static GstGhostPad * +_create_video_mixer_input (GstGLMixerBin * self, GstPad * mixer_pad) +{ + GstGLVideoMixerInput *input = + g_object_new (gst_gl_video_mixer_input_get_type (), "name", + GST_OBJECT_NAME (mixer_pad), "direction", GST_PAD_DIRECTION (mixer_pad), + NULL); + + if (!gst_ghost_pad_construct (GST_GHOST_PAD (input))) { + gst_object_unref (input); + return NULL; + } + + gst_pad_set_chain_function (GST_PAD (input), gst_gl_video_mixer_input_chain); + + input->mixer_pad = mixer_pad; + + return GST_GHOST_PAD (input); +} + +enum +{ + PROP_BIN_0, + PROP_BIN_BACKGROUND, +}; +#define DEFAULT_BACKGROUND GST_GL_VIDEO_MIXER_BACKGROUND_CHECKER + +static void gst_gl_video_mixer_bin_get_property (GObject * object, + guint prop_id, GValue * value, GParamSpec * pspec); +static void gst_gl_video_mixer_bin_set_property (GObject * object, + guint prop_id, const GValue * value, GParamSpec * pspec); + +typedef GstGLMixerBin GstGLVideoMixerBin; +typedef GstGLMixerBinClass GstGLVideoMixerBinClass; + +G_DEFINE_TYPE (GstGLVideoMixerBin, gst_gl_video_mixer_bin, + GST_TYPE_GL_MIXER_BIN); + +static void +gst_gl_video_mixer_bin_init (GstGLVideoMixerBin * self) +{ + GstGLMixerBin *mix_bin = GST_GL_MIXER_BIN (self); + + gst_gl_mixer_bin_finish_init_with_element (mix_bin, + g_object_new (GST_TYPE_GL_VIDEO_MIXER, NULL)); +} + +static void +gst_gl_video_mixer_bin_class_init (GstGLVideoMixerBinClass * klass) +{ + GstGLMixerBinClass *mixer_class = GST_GL_MIXER_BIN_CLASS (klass); + GObjectClass *gobject_class = G_OBJECT_CLASS (klass); + + mixer_class->create_input_pad = _create_video_mixer_input; + + gobject_class->set_property = gst_gl_video_mixer_bin_set_property; + gobject_class->get_property = gst_gl_video_mixer_bin_get_property; + + g_object_class_install_property (gobject_class, PROP_BIN_BACKGROUND, + g_param_spec_enum ("background", "Background", "Background type", + GST_GL_TYPE_VIDEO_MIXER_BACKGROUND, + DEFAULT_BACKGROUND, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); +} + +static void +gst_gl_video_mixer_bin_get_property (GObject * object, guint prop_id, + GValue * value, GParamSpec * pspec) +{ + GstGLMixerBin *self = GST_GL_MIXER_BIN (object); + + if (self->mixer) + g_object_get_property (G_OBJECT (self->mixer), pspec->name, value); +} + +static void +gst_gl_video_mixer_bin_set_property (GObject * object, guint prop_id, + const GValue * value, GParamSpec * pspec) +{ + GstGLMixerBin *self = GST_GL_MIXER_BIN (object); + + if (self->mixer) + g_object_set_property (G_OBJECT (self->mixer), pspec->name, value); +} + enum { PROP_0, PROP_BACKGROUND, }; -#define DEFAULT_BACKGROUND GST_GL_VIDEO_MIXER_BACKGROUND_CHECKER #define DEBUG_INIT \ GST_DEBUG_CATEGORY_INIT (gst_gl_video_mixer_debug, "glvideomixer", 0, "glvideomixer element"); @@ -178,14 +435,7 @@ static void gst_gl_video_mixer_pad_set_property (GObject * object, guint prop_id, const GValue * value, GParamSpec * pspec); static void gst_gl_video_mixer_pad_get_property (GObject * object, guint prop_id, GValue * value, GParamSpec * pspec); -static GstBuffer *gst_gl_video_mixer_pad_upload_buffer (GstGLMixer * mix, - GstGLMixerFrameData * frame, GstBuffer * buffer); -#define DEFAULT_PAD_XPOS 0 -#define DEFAULT_PAD_YPOS 0 -#define DEFAULT_PAD_WIDTH 0 -#define DEFAULT_PAD_HEIGHT 0 -#define DEFAULT_PAD_ALPHA 1.0 enum { PROP_PAD_0, @@ -206,7 +456,6 @@ static void gst_gl_video_mixer_pad_class_init (GstGLVideoMixerPadClass * klass) { GObjectClass *gobject_class = (GObjectClass *) klass; - GstGLMixerPadClass *mix_pad_class = (GstGLMixerPadClass *) klass; gobject_class->set_property = gst_gl_video_mixer_pad_set_property; gobject_class->get_property = gst_gl_video_mixer_pad_get_property; @@ -231,8 +480,6 @@ gst_gl_video_mixer_pad_class_init (GstGLVideoMixerPadClass * klass) g_param_spec_double ("alpha", "Alpha", "Alpha of the picture", 0.0, 1.0, DEFAULT_PAD_ALPHA, G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); - - mix_pad_class->upload_buffer = gst_gl_video_mixer_pad_upload_buffer; } static void @@ -298,44 +545,6 @@ gst_gl_video_mixer_pad_set_property (GObject * object, guint prop_id, gst_object_unref (mix); } -static GstBuffer * -gst_gl_video_mixer_pad_upload_buffer (GstGLMixer * mix, - GstGLMixerFrameData * frame, GstBuffer * buffer) -{ - GstGLVideoMixerPad *pad = GST_GL_VIDEO_MIXER_PAD (frame->pad); - - if (pad->alpha > 0.0) { - return - GST_GL_MIXER_PAD_CLASS - (gst_gl_video_mixer_pad_parent_class)->upload_buffer (mix, frame, - buffer); - } - - return NULL; -} - -#define GST_GL_TYPE_VIDEO_MIXER_BACKGROUND (gst_gl_video_mixer_background_get_type()) -static GType -gst_gl_video_mixer_background_get_type (void) -{ - static GType mixer_background_type = 0; - - static const GEnumValue mixer_background[] = { - {GST_GL_VIDEO_MIXER_BACKGROUND_CHECKER, "Checker pattern", "checker"}, - {GST_GL_VIDEO_MIXER_BACKGROUND_BLACK, "Black", "black"}, - {GST_GL_VIDEO_MIXER_BACKGROUND_WHITE, "White", "white"}, - {GST_GL_VIDEO_MIXER_BACKGROUND_TRANSPARENT, - "Transparent Background to enable further compositing", "transparent"}, - {0, NULL, NULL}, - }; - - if (!mixer_background_type) { - mixer_background_type = - g_enum_register_static ("GstGLVideoMixerBackground", mixer_background); - } - return mixer_background_type; -} - static void gst_gl_video_mixer_class_init (GstGLVideoMixerClass * klass) { @@ -352,7 +561,7 @@ gst_gl_video_mixer_class_init (GstGLVideoMixerClass * klass) gst_element_class_set_metadata (element_class, "OpenGL video_mixer", "Filter/Effect/Video/Compositor", "OpenGL video_mixer", - "Julien Isorce "); + "Matthew Waters "); g_object_class_install_property (gobject_class, PROP_BACKGROUND, g_param_spec_enum ("background", "Background", "Background type", @@ -454,7 +663,9 @@ _update_caps (GstVideoAggregator * vagg, GstCaps * caps) } GST_OBJECT_UNLOCK (vagg); - ret = gst_gl_mixer_update_caps (GST_GL_MIXER (vagg), caps); + ret = + GST_VIDEO_AGGREGATOR_CLASS (gst_gl_video_mixer_parent_class)->update_caps + (vagg, caps); for (i = 0; i < gst_caps_get_size (ret); i++) { GstStructure *s = gst_caps_get_structure (ret, i); diff --git a/ext/gl/gstglvideomixer.h b/ext/gl/gstglvideomixer.h index b351888fce..967358baba 100644 --- a/ext/gl/gstglvideomixer.h +++ b/ext/gl/gstglvideomixer.h @@ -73,6 +73,7 @@ struct _GstGLVideoMixerClass }; GType gst_gl_video_mixer_get_type (void); +GType gst_gl_video_mixer_bin_get_type (void); G_END_DECLS From f19b8f766599bd5127b41b4d1f1300c099496d6c Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Tue, 3 Mar 2015 17:26:47 +1100 Subject: [PATCH 140/381] gl: store the list of contexts within gldisplay Removes the reliance on the allocation query to propogate GL contexts. Allows thread safely getting a context for the a specific thread. --- ext/gl/gstglbasemixer.c | 66 ++++++++--------------------------------- 1 file changed, 13 insertions(+), 53 deletions(-) diff --git a/ext/gl/gstglbasemixer.c b/ext/gl/gstglbasemixer.c index 39d6738d14..20cc3c5771 100644 --- a/ext/gl/gstglbasemixer.c +++ b/ext/gl/gstglbasemixer.c @@ -479,8 +479,6 @@ gst_gl_base_mixer_decide_allocation (GstGLBaseMixer * mix, GstQuery * query) { GstGLBaseMixerClass *mix_class = GST_GL_BASE_MIXER_GET_CLASS (mix); GError *error = NULL; - guint idx; - GstGLContext *other_context = NULL; gboolean ret = TRUE; if (!gst_gl_ensure_element_data (mix, &mix->display, @@ -491,58 +489,20 @@ gst_gl_base_mixer_decide_allocation (GstGLBaseMixer * mix, GstQuery * query) _find_local_gl_context (mix); - if (gst_query_find_allocation_meta (query, - GST_VIDEO_GL_TEXTURE_UPLOAD_META_API_TYPE, &idx)) { - GstGLContext *context; - const GstStructure *upload_meta_params; - gpointer handle; - gchar *type; - gchar *apis; - - gst_query_parse_nth_allocation_meta (query, idx, &upload_meta_params); - if (upload_meta_params) { - if (gst_structure_get (upload_meta_params, "gst.gl.GstGLContext", - GST_GL_TYPE_CONTEXT, &context, NULL) && context) { - GstGLContext *old = mix->context; - - mix->context = context; - if (old) - gst_object_unref (old); - } else if (gst_structure_get (upload_meta_params, "gst.gl.context.handle", - G_TYPE_POINTER, &handle, "gst.gl.context.type", G_TYPE_STRING, - &type, "gst.gl.context.apis", G_TYPE_STRING, &apis, NULL) - && handle) { - GstGLPlatform platform; - GstGLAPI gl_apis; - - GST_DEBUG ("got GL context handle 0x%p with type %s and apis %s", - handle, type, apis); - - platform = gst_gl_platform_from_string (type); - gl_apis = gst_gl_api_from_string (apis); - - if (gl_apis && platform) - other_context = - gst_gl_context_new_wrapped (mix->display, (guintptr) handle, - platform, gl_apis); - } - } - } - - if (mix->priv->other_context) { - if (!other_context) { - other_context = mix->priv->other_context; - } else { - GST_ELEMENT_WARNING (mix, LIBRARY, SETTINGS, - ("%s", "Cannot share with more than one GL context"), - ("%s", "Cannot share with more than one GL context")); - } - } - if (!mix->context) { - mix->context = gst_gl_context_new (mix->display); - if (!gst_gl_context_create (mix->context, other_context, &error)) - goto context_error; + do { + if (mix->context) + gst_object_unref (mix->context); + /* just get a GL context. we don't care */ + mix->context = + gst_gl_display_get_gl_context_for_thread (mix->display, NULL); + if (!mix->context) { + mix->context = gst_gl_context_new (mix->display); + if (!gst_gl_context_create (mix->context, mix->priv->other_context, + &error)) + goto context_error; + } + } while (!gst_gl_display_add_context (mix->display, mix->context)); } if (mix_class->decide_allocation) From b0ee70e77a4de9f285bc1f36430ecc494da5484a Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Fri, 13 Mar 2015 09:38:54 +0000 Subject: [PATCH 141/381] gl: retreive the gldisplay/app gl context as soon as possible fixes the usage of gst_gl_display_filter_gl_api --- ext/gl/gstglbasemixer.c | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/ext/gl/gstglbasemixer.c b/ext/gl/gstglbasemixer.c index 20cc3c5771..f2eaa2f271 100644 --- a/ext/gl/gstglbasemixer.c +++ b/ext/gl/gstglbasemixer.c @@ -45,6 +45,8 @@ static void gst_gl_base_mixer_pad_finalize (GObject * object); static void gst_gl_base_mixer_set_context (GstElement * element, GstContext * context); +static GstStateChangeReturn gst_gl_base_mixer_change_state (GstElement * + element, GstStateChange transition); enum { @@ -280,6 +282,7 @@ gst_gl_base_mixer_class_init (GstGLBaseMixerClass * klass) element_class->set_context = GST_DEBUG_FUNCPTR (gst_gl_base_mixer_set_context); + element_class->change_state = gst_gl_base_mixer_change_state; agg_class->sinkpads_type = GST_TYPE_GL_BASE_MIXER_PAD; agg_class->sink_query = gst_gl_base_mixer_sink_query; @@ -692,3 +695,38 @@ gst_gl_base_mixer_stop (GstAggregator * agg) return TRUE; } + +static GstStateChangeReturn +gst_gl_base_mixer_change_state (GstElement * element, GstStateChange transition) +{ + GstGLBaseMixer *mix = GST_GL_BASE_MIXER (element); + GstGLBaseMixerClass *mix_class = GST_GL_BASE_MIXER_GET_CLASS (mix); + GstStateChangeReturn ret = GST_STATE_CHANGE_SUCCESS; + + GST_DEBUG_OBJECT (mix, "changing state: %s => %s", + gst_element_state_get_name (GST_STATE_TRANSITION_CURRENT (transition)), + gst_element_state_get_name (GST_STATE_TRANSITION_NEXT (transition))); + + switch (transition) { + case GST_STATE_CHANGE_NULL_TO_READY: + if (!gst_gl_ensure_element_data (element, &mix->display, + &mix->priv->other_context)) + return GST_STATE_CHANGE_FAILURE; + + gst_gl_display_filter_gl_api (mix->display, mix_class->supported_gl_api); + break; + default: + break; + } + + ret = GST_ELEMENT_CLASS (parent_class)->change_state (element, transition); + if (ret == GST_STATE_CHANGE_FAILURE) + return ret; + + switch (transition) { + default: + break; + } + + return ret; +} From 4c7291082503872f22821ad6d84ad3100bf4013b Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Fri, 13 Mar 2015 10:25:35 +0000 Subject: [PATCH 142/381] gl: get the context from basemixer/basefilter --- ext/gl/gstglmixer.c | 7 ------- ext/gl/gstglmixer.h | 3 --- ext/gl/gstglmosaic.c | 17 ++++++++-------- ext/gl/gstglvideomixer.c | 43 +++++++++++++++++++++------------------- 4 files changed, 32 insertions(+), 38 deletions(-) diff --git a/ext/gl/gstglmixer.c b/ext/gl/gstglmixer.c index a1da5f4e63..20d858a15f 100644 --- a/ext/gl/gstglmixer.c +++ b/ext/gl/gstglmixer.c @@ -112,19 +112,14 @@ static gboolean _negotiated_caps (GstVideoAggregator * vagg, GstCaps * caps) { GstGLMixer *mix = GST_GL_MIXER (vagg); - GstGLMixerClass *mix_class = GST_GL_MIXER_GET_CLASS (mix); - GstGLBaseMixerClass *base_mix_class = GST_GL_BASE_MIXER_GET_CLASS (mix); gboolean ret; mix->priv->negotiated = TRUE; - base_mix_class->supported_gl_api = mix_class->supported_gl_api; gst_caps_replace (&mix->out_caps, caps); ret = GST_VIDEO_AGGREGATOR_CLASS (parent_class)->negotiated_caps (vagg, caps); - mix->context = GST_GL_BASE_MIXER (mix)->context; - return ret; } @@ -389,7 +384,6 @@ gst_gl_mixer_class_init (GstGLMixerClass * klass) g_type_class_ref (GST_TYPE_GL_MIXER_PAD); klass->set_caps = NULL; - klass->supported_gl_api = GST_GL_API_ANY; } static void @@ -537,7 +531,6 @@ gst_gl_mixer_decide_allocation (GstGLBaseMixer * base_mix, GstQuery * query) gst_query_parse_allocation (query, &caps, NULL); - mix->context = context; if (mixer_class->set_caps) mixer_class->set_caps (mix, caps); diff --git a/ext/gl/gstglmixer.h b/ext/gl/gstglmixer.h index 5b334750e4..01eed34f20 100644 --- a/ext/gl/gstglmixer.h +++ b/ext/gl/gstglmixer.h @@ -85,8 +85,6 @@ struct _GstGLMixer { GstGLBaseMixer vaggregator; - GstGLContext *context; - GPtrArray *array_buffers; GPtrArray *frames; @@ -101,7 +99,6 @@ struct _GstGLMixer struct _GstGLMixerClass { GstGLBaseMixerClass parent_class; - GstGLAPI supported_gl_api; GstGLMixerSetCaps set_caps; GstGLMixerReset reset; diff --git a/ext/gl/gstglmosaic.c b/ext/gl/gstglmosaic.c index e8daef9f6d..726674b4df 100644 --- a/ext/gl/gstglmosaic.c +++ b/ext/gl/gstglmosaic.c @@ -135,7 +135,7 @@ gst_gl_mosaic_class_init (GstGLMosaicClass * klass) GST_GL_MIXER_CLASS (klass)->reset = gst_gl_mosaic_reset; GST_GL_MIXER_CLASS (klass)->process_textures = gst_gl_mosaic_process_textures; - GST_GL_MIXER_CLASS (klass)->supported_gl_api = GST_GL_API_OPENGL; + GST_GL_BASE_MIXER_CLASS (klass)->supported_gl_api = GST_GL_API_OPENGL; } static void @@ -180,7 +180,8 @@ gst_gl_mosaic_reset (GstGLMixer * mixer) //blocking call, wait the opengl thread has destroyed the shader if (mosaic->shader) - gst_gl_context_del_shader (mixer->context, mosaic->shader); + gst_gl_context_del_shader (GST_GL_BASE_MIXER (mixer)->context, + mosaic->shader); mosaic->shader = NULL; } @@ -190,8 +191,8 @@ gst_gl_mosaic_init_shader (GstGLMixer * mixer, GstCaps * outcaps) GstGLMosaic *mosaic = GST_GL_MOSAIC (mixer); //blocking call, wait the opengl thread has compiled the shader - return gst_gl_context_gen_shader (mixer->context, mosaic_v_src, mosaic_f_src, - &mosaic->shader); + return gst_gl_context_gen_shader (GST_GL_BASE_MIXER (mixer)->context, + mosaic_v_src, mosaic_f_src, &mosaic->shader); } static gboolean @@ -203,7 +204,7 @@ gst_gl_mosaic_process_textures (GstGLMixer * mix, GPtrArray * frames, mosaic->input_frames = frames; //blocking call, use a FBO - gst_gl_context_use_fbo_v2 (mix->context, + gst_gl_context_use_fbo_v2 (GST_GL_BASE_MIXER (mix)->context, GST_VIDEO_INFO_WIDTH (&GST_VIDEO_AGGREGATOR (mix)->info), GST_VIDEO_INFO_HEIGHT (&GST_VIDEO_AGGREGATOR (mix)->info), mix->fbo, mix->depthbuffer, out_tex, gst_gl_mosaic_callback, (gpointer) mosaic); @@ -217,7 +218,7 @@ gst_gl_mosaic_callback (gpointer stuff) { GstGLMosaic *mosaic = GST_GL_MOSAIC (stuff); GstGLMixer *mixer = GST_GL_MIXER (mosaic); - GstGLFuncs *gl = mixer->context->gl_vtable; + GstGLFuncs *gl = GST_GL_BASE_MIXER (mixer)->context->gl_vtable; static GLfloat xrot = 0; static GLfloat yrot = 0; @@ -239,7 +240,7 @@ gst_gl_mosaic_callback (gpointer stuff) guint count = 0; - gst_gl_context_clear_shader (mixer->context); + gst_gl_context_clear_shader (GST_GL_BASE_MIXER (mixer)->context); gl->BindTexture (GL_TEXTURE_2D, 0); gl->Disable (GL_TEXTURE_2D); @@ -344,7 +345,7 @@ gst_gl_mosaic_callback (gpointer stuff) gl->Disable (GL_DEPTH_TEST); - gst_gl_context_clear_shader (mixer->context); + gst_gl_context_clear_shader (GST_GL_BASE_MIXER (mixer)->context); xrot += 0.6f; yrot += 0.4f; diff --git a/ext/gl/gstglvideomixer.c b/ext/gl/gstglvideomixer.c index ad73404740..c399e924e1 100644 --- a/ext/gl/gstglvideomixer.c +++ b/ext/gl/gstglvideomixer.c @@ -577,7 +577,7 @@ gst_gl_video_mixer_class_init (GstGLVideoMixerClass * klass) agg_class->sinkpads_type = GST_TYPE_GL_VIDEO_MIXER_PAD; - GST_GL_MIXER_CLASS (klass)->supported_gl_api = + GST_GL_BASE_MIXER_CLASS (klass)->supported_gl_api = GST_GL_API_OPENGL | GST_GL_API_OPENGL3 | GST_GL_API_GLES2; } @@ -680,7 +680,7 @@ _update_caps (GstVideoAggregator * vagg, GstCaps * caps) static gboolean _reset_pad_gl (GstAggregator * agg, GstAggregatorPad * aggpad, gpointer udata) { - const GstGLFuncs *gl = GST_GL_MIXER (agg)->context->gl_vtable; + const GstGLFuncs *gl = GST_GL_BASE_MIXER (agg)->context->gl_vtable; GstGLVideoMixerPad *pad = GST_GL_VIDEO_MIXER_PAD (aggpad); if (pad->vertex_buffer) { @@ -694,7 +694,7 @@ _reset_pad_gl (GstAggregator * agg, GstAggregatorPad * aggpad, gpointer udata) static void _reset_gl (GstGLContext * context, GstGLVideoMixer * video_mixer) { - const GstGLFuncs *gl = GST_GL_MIXER (video_mixer)->context->gl_vtable; + const GstGLFuncs *gl = GST_GL_BASE_MIXER (video_mixer)->context->gl_vtable; if (video_mixer->vao) { gl->DeleteVertexArrays (1, &video_mixer->vao); @@ -713,16 +713,17 @@ gst_gl_video_mixer_reset (GstGLMixer * mixer) video_mixer->input_frames = NULL; if (video_mixer->shader) - gst_gl_context_del_shader (mixer->context, video_mixer->shader); + gst_gl_context_del_shader (GST_GL_BASE_MIXER (mixer)->context, + video_mixer->shader); video_mixer->shader = NULL; if (video_mixer->checker) - gst_gl_context_del_shader (mixer->context, video_mixer->checker); + gst_gl_context_del_shader (GST_GL_BASE_MIXER (mixer)->context, + video_mixer->checker); video_mixer->checker = NULL; - if (mixer->context) - gst_gl_context_thread_add (mixer->context, - (GstGLContextThreadFunc) _reset_gl, mixer); + gst_gl_context_thread_add (GST_GL_BASE_MIXER (mixer)->context, + (GstGLContextThreadFunc) _reset_gl, mixer); } static gboolean @@ -731,10 +732,11 @@ gst_gl_video_mixer_init_shader (GstGLMixer * mixer, GstCaps * outcaps) GstGLVideoMixer *video_mixer = GST_GL_VIDEO_MIXER (mixer); if (video_mixer->shader) - gst_gl_context_del_shader (mixer->context, video_mixer->shader); + gst_gl_context_del_shader (GST_GL_BASE_MIXER (mixer)->context, + video_mixer->shader); - return gst_gl_context_gen_shader (mixer->context, video_mixer_v_src, - video_mixer_f_src, &video_mixer->shader); + return gst_gl_context_gen_shader (GST_GL_BASE_MIXER (mixer)->context, + video_mixer_v_src, video_mixer_f_src, &video_mixer->shader); } static gboolean @@ -745,7 +747,7 @@ gst_gl_video_mixer_process_textures (GstGLMixer * mix, GPtrArray * frames, video_mixer->input_frames = frames; - gst_gl_context_use_fbo_v2 (mix->context, + gst_gl_context_use_fbo_v2 (GST_GL_BASE_MIXER (mix)->context, GST_VIDEO_INFO_WIDTH (&GST_VIDEO_AGGREGATOR (mix)->info), GST_VIDEO_INFO_HEIGHT (&GST_VIDEO_AGGREGATOR (mix)->info), mix->fbo, mix->depthbuffer, @@ -758,7 +760,7 @@ static gboolean _draw_checker_background (GstGLVideoMixer * video_mixer) { GstGLMixer *mixer = GST_GL_MIXER (video_mixer); - const GstGLFuncs *gl = mixer->context->gl_vtable; + const GstGLFuncs *gl = GST_GL_BASE_MIXER (mixer)->context->gl_vtable; gint attr_position_loc = 0; const GLushort indices[] = { @@ -775,8 +777,8 @@ _draw_checker_background (GstGLVideoMixer * video_mixer) /* *INDENT-ON* */ if (!video_mixer->checker) { - if (!gst_gl_context_gen_shader (mixer->context, checker_v_src, - checker_f_src, &video_mixer->checker)) + if (!gst_gl_context_gen_shader (GST_GL_BASE_MIXER (mixer)->context, + checker_v_src, checker_f_src, &video_mixer->checker)) return FALSE; } @@ -810,7 +812,7 @@ static gboolean _draw_background (GstGLVideoMixer * video_mixer) { GstGLMixer *mixer = GST_GL_MIXER (video_mixer); - const GstGLFuncs *gl = mixer->context->gl_vtable; + const GstGLFuncs *gl = GST_GL_BASE_MIXER (mixer)->context->gl_vtable; switch (video_mixer->background) { case GST_GL_VIDEO_MIXER_BACKGROUND_BLACK: @@ -841,7 +843,7 @@ gst_gl_video_mixer_callback (gpointer stuff) { GstGLVideoMixer *video_mixer = GST_GL_VIDEO_MIXER (stuff); GstGLMixer *mixer = GST_GL_MIXER (video_mixer); - GstGLFuncs *gl = mixer->context->gl_vtable; + GstGLFuncs *gl = GST_GL_BASE_MIXER (mixer)->context->gl_vtable; GLint attr_position_loc = 0; GLint attr_texture_loc = 0; @@ -857,9 +859,10 @@ gst_gl_video_mixer_callback (gpointer stuff) out_width = GST_VIDEO_INFO_WIDTH (&GST_VIDEO_AGGREGATOR (stuff)->info); out_height = GST_VIDEO_INFO_HEIGHT (&GST_VIDEO_AGGREGATOR (stuff)->info); - gst_gl_context_clear_shader (mixer->context); + gst_gl_context_clear_shader (GST_GL_BASE_MIXER (mixer)->context); gl->BindTexture (GL_TEXTURE_2D, 0); - if (gst_gl_context_get_gl_api (mixer->context) & GST_GL_API_OPENGL) + if (gst_gl_context_get_gl_api (GST_GL_BASE_MIXER (mixer)->context) & + GST_GL_API_OPENGL) gl->Disable (GL_TEXTURE_2D); gl->Disable (GL_DEPTH_TEST); @@ -986,5 +989,5 @@ gst_gl_video_mixer_callback (gpointer stuff) gl->Disable (GL_BLEND); - gst_gl_context_clear_shader (mixer->context); + gst_gl_context_clear_shader (GST_GL_BASE_MIXER (mixer)->context); } From 875e5c0320e3a4d3e9f6220f3f6e47c2dd7dcf70 Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Sat, 14 Mar 2015 16:25:29 +0000 Subject: [PATCH 143/381] glsyncmeta: make context to wait and set sync explicit otherwise we may wait on a sync object in same context by accident --- ext/gl/gstglmixer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/gl/gstglmixer.c b/ext/gl/gstglmixer.c index 20d858a15f..51bd517a68 100644 --- a/ext/gl/gstglmixer.c +++ b/ext/gl/gstglmixer.c @@ -631,7 +631,7 @@ gst_gl_mixer_process_textures (GstGLMixer * mix, GstBuffer * outbuf) sync_meta = gst_buffer_get_gl_sync_meta (vaggpad->buffer); if (sync_meta) - gst_gl_sync_meta_wait (sync_meta); + gst_gl_sync_meta_wait (sync_meta, GST_GL_BASE_MIXER (mix)->context); if (gst_video_frame_map (&gl_frame, &gl_info, vaggpad->buffer, GST_MAP_READ | GST_MAP_GL)) { From 6a1f97cd508352b6cb1942be816b4719b4ff2856 Mon Sep 17 00:00:00 2001 From: Jan Schmidt Date: Mon, 16 Mar 2015 00:22:14 +1100 Subject: [PATCH 144/381] compositor: Fix blending functions Correctly calculate alpha in a few places by dividing by 255, not 256. Fix the argb and bgra blending functions to avoid an off-by-one error in the calculations, so painting with alpha = 0xff doesn't ever bleed through from behind --- gst/compositor/compositororc.orc | 41 ++++++++++++++++++++++---------- 1 file changed, 29 insertions(+), 12 deletions(-) diff --git a/gst/compositor/compositororc.orc b/gst/compositor/compositororc.orc index 5a348b2539..a9368b7343 100644 --- a/gst/compositor/compositororc.orc +++ b/gst/compositor/compositororc.orc @@ -35,6 +35,7 @@ convsuswb d1, t2 .source 4 s guint8 .param 2 alpha .temp 4 t +.temp 4 t2 .temp 2 tw .temp 1 tb .temp 4 a @@ -49,14 +50,26 @@ convwb tb, tw splatbl a, tb x4 convubw a_wide, a x4 mullw a_wide, a_wide, alpha -x4 shruw a_wide, a_wide, 8 +x4 div255w a_wide, a_wide + +# dest pixel into t2 +loadl t2, d + +# t = s - d (as bytes) +x4 subb t, t, t2 + +# s_wide = (uint16)(t) * alpha x4 convubw s_wide, t -loadl t, d -x4 convubw d_wide, t -x4 subw s_wide, s_wide, d_wide x4 mullw s_wide, s_wide, a_wide + +# s_wide /= 255 x4 div255w s_wide, s_wide + +# d_wide = (uint16)(dest) + s_wide +x4 convubw d_wide, t2 x4 addw d_wide, d_wide, s_wide + +# Set output alpha to 0xff and store x4 convwb t, d_wide orl t, t, a_alpha storel d, t @@ -83,13 +96,17 @@ convwb tb, tw splatbl a, tb x4 convubw a_wide, a x4 mullw a_wide, a_wide, alpha -x4 shruw a_wide, a_wide, 8 +x4 div255w a_wide, a_wide + +loadl t2, d +# t = s - d (as bytes) +x4 subb t, t, t2 + x4 convubw s_wide, t -loadl t, d -x4 convubw d_wide, t -x4 subw s_wide, s_wide, d_wide x4 mullw s_wide, s_wide, a_wide x4 div255w s_wide, s_wide + +x4 convubw d_wide, t2 x4 addw d_wide, d_wide, s_wide x4 convwb t, d_wide orl t, t, a_alpha @@ -114,14 +131,14 @@ storel d, t .const 4 a_alpha 0x000000ff .const 4 a_alpha_inv 0xffffff00 -# calc source alpha as alpha_s = alpha_s * alpha / 256 +# calc source alpha as alpha_s = alpha_s * alpha / 255 loadl t, s convlw tw, t convwb tb, tw splatbl a, tb x4 convubw alpha_s, a x4 mullw alpha_s, alpha_s, alpha -x4 shruw alpha_s, alpha_s, 8 +x4 div255w alpha_s, alpha_s x4 convubw s_wide, t x4 mullw s_wide, s_wide, alpha_s @@ -175,7 +192,7 @@ storel d, t .const 4 a_alpha 0xff000000 .const 4 a_alpha_inv 0x00ffffff -# calc source alpha as alpha_s = alpha_s * alpha / 256 +# calc source alpha as alpha_s = alpha_s * alpha / 255 loadl t, s shrul t2, t, 24 convlw tw, t2 @@ -183,7 +200,7 @@ convwb tb, tw splatbl a, tb x4 convubw alpha_s, a x4 mullw alpha_s, alpha_s, alpha -x4 shruw alpha_s, alpha_s, 8 +x4 div255w alpha_s, alpha_s x4 convubw s_wide, t x4 mullw s_wide, s_wide, alpha_s From d9a96ac2ecda57d43bb7b96ac600ba9898cfdb47 Mon Sep 17 00:00:00 2001 From: Jan Schmidt Date: Mon, 16 Mar 2015 03:53:33 +1100 Subject: [PATCH 145/381] compositor: Revert most of previous patch. The calculation doesn't produce the same results. Keep just the change to divide alpha by 255 instead of 256, for slightly better accuracy --- gst/compositor/compositororc.orc | 28 +++++++--------------------- 1 file changed, 7 insertions(+), 21 deletions(-) diff --git a/gst/compositor/compositororc.orc b/gst/compositor/compositororc.orc index a9368b7343..234ec21128 100644 --- a/gst/compositor/compositororc.orc +++ b/gst/compositor/compositororc.orc @@ -17,7 +17,7 @@ copyl d1, s1 .param 2 p1 .temp 2 t1 .temp 2 t2 -.const 1 c1 8 +.const 1 c1 8 convubw t1, d1 convubw t2, s1 @@ -35,7 +35,6 @@ convsuswb d1, t2 .source 4 s guint8 .param 2 alpha .temp 4 t -.temp 4 t2 .temp 2 tw .temp 1 tb .temp 4 a @@ -51,25 +50,14 @@ splatbl a, tb x4 convubw a_wide, a x4 mullw a_wide, a_wide, alpha x4 div255w a_wide, a_wide - -# dest pixel into t2 -loadl t2, d - -# t = s - d (as bytes) -x4 subb t, t, t2 - -# s_wide = (uint16)(t) * alpha x4 convubw s_wide, t +loadl t, d +x4 convubw d_wide, t +x4 subw s_wide, s_wide, d_wide x4 mullw s_wide, s_wide, a_wide -# s_wide /= 255 x4 div255w s_wide, s_wide - -# d_wide = (uint16)(dest) + s_wide -x4 convubw d_wide, t2 x4 addw d_wide, d_wide, s_wide - -# Set output alpha to 0xff and store x4 convwb t, d_wide orl t, t, a_alpha storel d, t @@ -98,15 +86,13 @@ x4 convubw a_wide, a x4 mullw a_wide, a_wide, alpha x4 div255w a_wide, a_wide -loadl t2, d -# t = s - d (as bytes) -x4 subb t, t, t2 - x4 convubw s_wide, t +loadl t, d +x4 convubw d_wide, t +x4 subw s_wide, s_wide, d_wide x4 mullw s_wide, s_wide, a_wide x4 div255w s_wide, s_wide -x4 convubw d_wide, t2 x4 addw d_wide, d_wide, s_wide x4 convwb t, d_wide orl t, t, a_alpha From af17a2cd4fc050b0fbecb811f812963851b3485c Mon Sep 17 00:00:00 2001 From: Nirbheek Chauhan Date: Tue, 24 Mar 2015 19:04:26 +0530 Subject: [PATCH 146/381] videoaggregator: Check if there's a previous buffer to 'keep' If we want to keep a previous buffer but there's no previous buffer, we actually need more data instead. --- gst-libs/gst/video/gstvideoaggregator.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index 24ab93f47e..272a35bba0 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -1006,8 +1006,14 @@ gst_videoaggregator_fill_queues (GstVideoAggregator * vagg, gst_segment_to_running_time (&segment, GST_FORMAT_TIME, start_time); if (start_time >= output_end_time) { - GST_DEBUG_OBJECT (pad, "buffer duration is -1, start_time >= " - "output_end_time. Keeping previous buffer"); + if (pad->buffer) { + GST_DEBUG_OBJECT (pad, "buffer duration is -1, start_time >= " + "output_end_time. Keeping previous buffer"); + } else { + GST_DEBUG_OBJECT (pad, "buffer duration is -1, start_time >= " + "output_end_time. No previous buffer, need more data"); + need_more_data = TRUE; + } gst_buffer_unref (buf); continue; } else if (start_time < output_start_time) { From 9a73ffcef33951dc5faef69d3c3fb4bfc18e3d7d Mon Sep 17 00:00:00 2001 From: Edward Hervey Date: Tue, 31 Mar 2015 16:27:00 +0200 Subject: [PATCH 147/381] glvideomixer: Don't use context if not present Avoids assertions at runtime --- ext/gl/gstglvideomixer.c | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/ext/gl/gstglvideomixer.c b/ext/gl/gstglvideomixer.c index c399e924e1..cbba72a5d7 100644 --- a/ext/gl/gstglvideomixer.c +++ b/ext/gl/gstglvideomixer.c @@ -709,21 +709,23 @@ static void gst_gl_video_mixer_reset (GstGLMixer * mixer) { GstGLVideoMixer *video_mixer = GST_GL_VIDEO_MIXER (mixer); + GstGLContext *context = GST_GL_BASE_MIXER (mixer)->context; video_mixer->input_frames = NULL; + GST_DEBUG_OBJECT (mixer, "context:%p", context); + if (video_mixer->shader) - gst_gl_context_del_shader (GST_GL_BASE_MIXER (mixer)->context, - video_mixer->shader); + gst_gl_context_del_shader (context, video_mixer->shader); video_mixer->shader = NULL; if (video_mixer->checker) - gst_gl_context_del_shader (GST_GL_BASE_MIXER (mixer)->context, - video_mixer->checker); + gst_gl_context_del_shader (context, video_mixer->checker); video_mixer->checker = NULL; - gst_gl_context_thread_add (GST_GL_BASE_MIXER (mixer)->context, - (GstGLContextThreadFunc) _reset_gl, mixer); + if (GST_GL_BASE_MIXER (mixer)->context) + gst_gl_context_thread_add (context, (GstGLContextThreadFunc) _reset_gl, + mixer); } static gboolean From 26d3057d989cae5935a0dc5976966cb3cb4d1d7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Olivier=20Cr=C3=AAte?= Date: Fri, 6 Mar 2015 21:12:52 -0500 Subject: [PATCH 148/381] aggregator: Query latency on first incoming buffer. And keep on querying upstream until we get a reply. Also, the _get_latency_unlocked() method required being calld with a private lock, so removed the _unlocked() variant from the API. And it now returns GST_CLOCK_TIME_NONE when the element is not live as we think that 0 upstream latency is possible. https://bugzilla.gnome.org/show_bug.cgi?id=745768 --- gst-libs/gst/video/gstvideoaggregator.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index 272a35bba0..b1cfc835ad 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -880,9 +880,11 @@ gst_videoaggregator_update_qos (GstVideoAggregator * vagg, gdouble proportion, GST_TIME_FORMAT, proportion, (diff < 0) ? "-" : "", GST_TIME_ARGS (ABS (diff)), GST_TIME_ARGS (timestamp)); + live = + GST_CLOCK_TIME_IS_VALID (gst_aggregator_get_latency (GST_AGGREGATOR + (vagg))); + GST_OBJECT_LOCK (vagg); - gst_aggregator_get_latency_unlocked (GST_AGGREGATOR (vagg), &live, NULL, - NULL); vagg->priv->proportion = proportion; if (G_LIKELY (timestamp != GST_CLOCK_TIME_NONE)) { From 63c776adee1a6ec2f8b199ad260df38a2264c37f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Wed, 8 Apr 2015 19:29:40 -0700 Subject: [PATCH 149/381] videoaggregator: Remove broken timestamps-going-backwards check This would've also triggered if for some reason the segment was updated in such a way that PTS went backwards, but the running time increased. Like what happens when non-flushing seeks are done. We're doing a proper buffer-from-the-past check a few lines below based on the running time, which is the only time we should care about here. --- gst-libs/gst/video/gstvideoaggregator.c | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index b1cfc835ad..0f845f9415 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -990,16 +990,6 @@ gst_videoaggregator_fill_queues (GstVideoAggregator * vagg, vinfo = &pad->info; /* FIXME: Make all this work with negative rates */ - - if ((start_time < GST_BUFFER_TIMESTAMP (buf)) - || (pad->buffer && start_time < GST_BUFFER_TIMESTAMP (pad->buffer))) { - GST_DEBUG_OBJECT (pad, "Buffer from the past, dropping"); - gst_buffer_unref (buf); - gst_aggregator_pad_drop_buffer (bpad); - need_more_data = TRUE; - continue; - } - end_time = GST_BUFFER_DURATION (buf); if (end_time == -1) { From 78a1841c86f99cf36844658f1c6e6a152862990f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Fri, 17 Apr 2015 14:09:47 +0200 Subject: [PATCH 150/381] gl: Remove some empty ::finalize() implementations --- ext/gl/gstglbasemixer.c | 9 --------- ext/gl/gstglmixer.c | 9 --------- 2 files changed, 18 deletions(-) diff --git a/ext/gl/gstglbasemixer.c b/ext/gl/gstglbasemixer.c index f2eaa2f271..64c019ca3a 100644 --- a/ext/gl/gstglbasemixer.c +++ b/ext/gl/gstglbasemixer.c @@ -41,7 +41,6 @@ static void gst_gl_base_mixer_pad_get_property (GObject * object, guint prop_id, GValue * value, GParamSpec * pspec); static void gst_gl_base_mixer_pad_set_property (GObject * object, guint prop_id, const GValue * value, GParamSpec * pspec); -static void gst_gl_base_mixer_pad_finalize (GObject * object); static void gst_gl_base_mixer_set_context (GstElement * element, GstContext * context); @@ -81,19 +80,11 @@ gst_gl_base_mixer_pad_class_init (GstGLBaseMixerPadClass * klass) gobject_class->set_property = gst_gl_base_mixer_pad_set_property; gobject_class->get_property = gst_gl_base_mixer_pad_get_property; - gobject_class->finalize = gst_gl_base_mixer_pad_finalize; - vaggpad_class->set_info = NULL; vaggpad_class->prepare_frame = NULL; vaggpad_class->clean_frame = NULL; } -static void -gst_gl_base_mixer_pad_finalize (GObject * object) -{ - G_OBJECT_CLASS (gst_gl_base_mixer_pad_parent_class)->finalize (object); -} - static void gst_gl_base_mixer_pad_get_property (GObject * object, guint prop_id, GValue * value, GParamSpec * pspec) diff --git a/ext/gl/gstglmixer.c b/ext/gl/gstglmixer.c index 51bd517a68..5becd24cd3 100644 --- a/ext/gl/gstglmixer.c +++ b/ext/gl/gstglmixer.c @@ -42,7 +42,6 @@ static void gst_gl_mixer_pad_get_property (GObject * object, guint prop_id, GValue * value, GParamSpec * pspec); static void gst_gl_mixer_pad_set_property (GObject * object, guint prop_id, const GValue * value, GParamSpec * pspec); -static void gst_gl_mixer_pad_finalize (GObject * object); enum { @@ -73,19 +72,11 @@ gst_gl_mixer_pad_class_init (GstGLMixerPadClass * klass) gobject_class->set_property = gst_gl_mixer_pad_set_property; gobject_class->get_property = gst_gl_mixer_pad_get_property; - gobject_class->finalize = gst_gl_mixer_pad_finalize; - vaggpad_class->set_info = NULL; vaggpad_class->prepare_frame = NULL; vaggpad_class->clean_frame = NULL; } -static void -gst_gl_mixer_pad_finalize (GObject * object) -{ - G_OBJECT_CLASS (gst_gl_mixer_pad_parent_class)->finalize (object); -} - static void gst_gl_mixer_pad_get_property (GObject * object, guint prop_id, GValue * value, GParamSpec * pspec) From 293d6a7ebe71221b97a800e2b1bb7464474842a1 Mon Sep 17 00:00:00 2001 From: Guillaume Desmottes Date: Wed, 15 Apr 2015 15:16:33 +0200 Subject: [PATCH 151/381] glmixer: pass the proper free function to frames and buffers array 'array_buffers' contain borrowed GstBuffer and so shouldn't have a free function. 'frames' is the one containing GstGLMixerFrameData and so should use _free_glmixer_frame_data as free function. Fix GstGLMixerFrameData leaks with the validate.file.glvideomixer.simple.play_15s.synchronized scenario. https://bugzilla.gnome.org/show_bug.cgi?id=747913 Signed-off-by: Guillaume Desmottes --- ext/gl/gstglmixer.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ext/gl/gstglmixer.c b/ext/gl/gstglmixer.c index 5becd24cd3..4e45124ed6 100644 --- a/ext/gl/gstglmixer.c +++ b/ext/gl/gstglmixer.c @@ -745,9 +745,9 @@ gst_gl_mixer_start (GstAggregator * agg) GstElement *element = GST_ELEMENT (agg); GST_OBJECT_LOCK (mix); - mix->array_buffers = g_ptr_array_new_full (element->numsinkpads, + mix->array_buffers = g_ptr_array_new_full (element->numsinkpads, NULL); + mix->frames = g_ptr_array_new_full (element->numsinkpads, (GDestroyNotify) _free_glmixer_frame_data); - mix->frames = g_ptr_array_new_full (element->numsinkpads, NULL); g_ptr_array_set_size (mix->array_buffers, element->numsinkpads); g_ptr_array_set_size (mix->frames, element->numsinkpads); From 516dc5e302d2b4f0c27ca311db4e38578b7235a6 Mon Sep 17 00:00:00 2001 From: Guillaume Desmottes Date: Thu, 16 Apr 2015 14:21:16 +0200 Subject: [PATCH 152/381] glmixer: unref owned caps when finalizing the mixer Fix a caps leak with the validate.file.glvideomixer.simple.play_15s.synchronized scenario. https://bugzilla.gnome.org/show_bug.cgi?id=747915 Signed-off-by: Guillaume Desmottes --- ext/gl/gstglmixer.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ext/gl/gstglmixer.c b/ext/gl/gstglmixer.c index 4e45124ed6..ab667a811b 100644 --- a/ext/gl/gstglmixer.c +++ b/ext/gl/gstglmixer.c @@ -404,6 +404,9 @@ gst_gl_mixer_finalize (GObject * object) GstGLMixer *mix = GST_GL_MIXER (object); GstGLMixerPrivate *priv = mix->priv; + if (mix->out_caps) + gst_caps_unref (mix->out_caps); + g_mutex_clear (&priv->gl_resource_lock); g_cond_clear (&priv->gl_resource_cond); G_OBJECT_CLASS (parent_class)->finalize (object); From f5f632a50312a90ae4a4e821bc65298639896034 Mon Sep 17 00:00:00 2001 From: Guillaume Desmottes Date: Thu, 16 Apr 2015 14:17:04 +0200 Subject: [PATCH 153/381] videoaggregator: fix caps leak when early returning https://bugzilla.gnome.org/show_bug.cgi?id=747993 Signed-off-by: Guillaume Desmottes --- gst-libs/gst/video/gstvideoaggregator.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index 0f845f9415..965aafcd43 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -734,10 +734,11 @@ gst_videoaggregator_update_src_caps (GstVideoAggregator * vagg) gst_caps_unref (caps); gst_caps_unref (peercaps); - caps = tmp; + caps = tmp; /* pass ownership */ if (gst_caps_is_empty (caps)) { GST_DEBUG_OBJECT (vagg, "empty caps"); ret = FALSE; + gst_caps_unref (caps); goto done; } From 9827138061a6e10a629c1beb4fbfe8e16d32e385 Mon Sep 17 00:00:00 2001 From: Guillaume Desmottes Date: Wed, 15 Apr 2015 15:22:37 +0200 Subject: [PATCH 154/381] glmixer: fix caps leak in gst_gl_mixer_pad_sink_getcaps() Caps refcounting was all wrong in this function. Rewrote it and add some comments to make it clearer. Fix caps leaks with the validate.file.glvideomixer.simple.play_15s.synchronized scenario. https://bugzilla.gnome.org/show_bug.cgi?id=747915 Signed-off-by: Guillaume Desmottes --- ext/gl/gstglmixer.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/ext/gl/gstglmixer.c b/ext/gl/gstglmixer.c index ab667a811b..96d6e2ac42 100644 --- a/ext/gl/gstglmixer.c +++ b/ext/gl/gstglmixer.c @@ -213,27 +213,27 @@ gst_gl_mixer_pad_sink_getcaps (GstPad * pad, GstGLMixer * mix, GstCaps * filter) GstCaps *template_caps; GstCaps *filtered_caps; GstCaps *returned_caps; - gboolean had_current_caps = TRUE; template_caps = gst_pad_get_pad_template_caps (pad); sinkcaps = gst_pad_get_current_caps (pad); if (sinkcaps == NULL) { - had_current_caps = FALSE; - sinkcaps = template_caps; + sinkcaps = gst_caps_ref (template_caps); } else { - sinkcaps = gst_caps_merge (sinkcaps, template_caps); + sinkcaps = gst_caps_merge (sinkcaps, gst_caps_ref (template_caps)); } - filtered_caps = sinkcaps; - if (filter) + if (filter) { filtered_caps = gst_caps_intersect (sinkcaps, filter); + gst_caps_unref (sinkcaps); + } else { + filtered_caps = sinkcaps; /* pass ownership */ + } + returned_caps = gst_caps_intersect (filtered_caps, template_caps); - if (filter) - gst_caps_unref (filtered_caps); - if (had_current_caps) - gst_caps_unref (template_caps); + gst_caps_unref (template_caps); + gst_caps_unref (filtered_caps); GST_DEBUG_OBJECT (pad, "returning %" GST_PTR_FORMAT, returned_caps); From 66a5cad6c1f1feefca68e16b7c62916b25d7fd47 Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Tue, 21 Apr 2015 12:19:46 +1000 Subject: [PATCH 155/381] gldisplay: synchronize the searching and creation of GstGLContext's Ootherwise we could end up with multiple elements in different chains each creating a context. Fixes context creation with glvideomixer. --- ext/gl/gstglbasemixer.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ext/gl/gstglbasemixer.c b/ext/gl/gstglbasemixer.c index 64c019ca3a..448c6a6b7f 100644 --- a/ext/gl/gstglbasemixer.c +++ b/ext/gl/gstglbasemixer.c @@ -484,6 +484,7 @@ gst_gl_base_mixer_decide_allocation (GstGLBaseMixer * mix, GstQuery * query) _find_local_gl_context (mix); if (!mix->context) { + GST_OBJECT_LOCK (mix->display); do { if (mix->context) gst_object_unref (mix->context); @@ -497,6 +498,7 @@ gst_gl_base_mixer_decide_allocation (GstGLBaseMixer * mix, GstQuery * query) goto context_error; } } while (!gst_gl_display_add_context (mix->display, mix->context)); + GST_OBJECT_UNLOCK (mix->display); } if (mix_class->decide_allocation) From 52a40dfc48f96a7fafe0da63823ef583b77e4b48 Mon Sep 17 00:00:00 2001 From: Vineeth T M Date: Wed, 22 Apr 2015 18:54:45 +0900 Subject: [PATCH 156/381] glmixer: Possible null pointer dereference While printing error message when context fails, error variable is not being used anymore so it will lead to null pointer dereference https://bugzilla.gnome.org/show_bug.cgi?id=748287 --- ext/gl/gstglmixer.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/ext/gl/gstglmixer.c b/ext/gl/gstglmixer.c index 96d6e2ac42..8e35af10b0 100644 --- a/ext/gl/gstglmixer.c +++ b/ext/gl/gstglmixer.c @@ -502,7 +502,6 @@ gst_gl_mixer_decide_allocation (GstGLBaseMixer * base_mix, GstQuery * query) GstCaps *caps; guint min, max, size; gboolean update_pool; - GError *error = NULL; guint out_width, out_height; out_width = GST_VIDEO_INFO_WIDTH (&vagg->info); @@ -566,8 +565,7 @@ gst_gl_mixer_decide_allocation (GstGLBaseMixer * base_mix, GstQuery * query) context_error: { - GST_ELEMENT_ERROR (mix, RESOURCE, NOT_FOUND, ("%s", error->message), - (NULL)); + GST_ELEMENT_ERROR (mix, RESOURCE, NOT_FOUND, ("Context error"), (NULL)); return FALSE; } } From 8930bf7c07836afe1a859a36f698fb72393f5d34 Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Mon, 27 Apr 2015 15:20:56 +1000 Subject: [PATCH 157/381] gl: unref display/other-context in the correct place Otherwise state changes from PLAYING->READY->PAUSED will cause there to to be no display configured on the element. https://bugzilla.gnome.org/show_bug.cgi?id=748405 --- ext/gl/gstglbasemixer.c | 33 +++++++++++---------------------- 1 file changed, 11 insertions(+), 22 deletions(-) diff --git a/ext/gl/gstglbasemixer.c b/ext/gl/gstglbasemixer.c index 448c6a6b7f..0e670e044d 100644 --- a/ext/gl/gstglbasemixer.c +++ b/ext/gl/gstglbasemixer.c @@ -247,8 +247,6 @@ static gboolean gst_gl_base_mixer_set_allocation (GstGLBaseMixer * mix, GstBufferPool * pool, GstAllocator * allocator, GstAllocationParams * params, GstQuery * query); -static void gst_gl_base_mixer_finalize (GObject * object); - static void gst_gl_base_mixer_class_init (GstGLBaseMixerClass * klass) { @@ -266,8 +264,6 @@ gst_gl_base_mixer_class_init (GstGLBaseMixerClass * klass) g_type_class_add_private (klass, sizeof (GstGLBaseMixerPrivate)); - gobject_class->finalize = GST_DEBUG_FUNCPTR (gst_gl_base_mixer_finalize); - gobject_class->get_property = gst_gl_base_mixer_get_property; gobject_class->set_property = gst_gl_base_mixer_set_property; @@ -313,19 +309,6 @@ gst_gl_base_mixer_init (GstGLBaseMixer * mix) gst_gl_base_mixer_reset (mix); } -static void -gst_gl_base_mixer_finalize (GObject * object) -{ - GstGLBaseMixer *mix = GST_GL_BASE_MIXER (object); - - if (mix->priv->other_context) { - gst_object_unref (mix->priv->other_context); - mix->priv->other_context = NULL; - } - - G_OBJECT_CLASS (parent_class)->finalize (object); -} - static void gst_gl_base_mixer_set_context (GstElement * element, GstContext * context) { @@ -674,11 +657,6 @@ gst_gl_base_mixer_stop (GstAggregator * agg) mix->priv->pool = NULL; } - if (mix->display) { - gst_object_unref (mix->display); - mix->display = NULL; - } - if (mix->context) { gst_object_unref (mix->context); mix->context = NULL; @@ -717,6 +695,17 @@ gst_gl_base_mixer_change_state (GstElement * element, GstStateChange transition) return ret; switch (transition) { + case GST_STATE_CHANGE_READY_TO_NULL: + if (mix->priv->other_context) { + gst_object_unref (mix->priv->other_context); + mix->priv->other_context = NULL; + } + + if (mix->display) { + gst_object_unref (mix->display); + mix->display = NULL; + } + break; default: break; } From 993e682a4b4ffac1f0676228a9118e0f6d08b2d4 Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Tue, 28 Apr 2015 20:11:07 +1000 Subject: [PATCH 158/381] glupload: provide the sink template caps that could be used https://bugzilla.gnome.org/show_bug.cgi?id=746399 --- ext/gl/gstglmixerbin.c | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/ext/gl/gstglmixerbin.c b/ext/gl/gstglmixerbin.c index 2dada971d7..47fff2a542 100644 --- a/ext/gl/gstglmixerbin.c +++ b/ext/gl/gstglmixerbin.c @@ -110,12 +110,6 @@ static GstStaticPadTemplate src_factory = GST_STATIC_PAD_TEMPLATE ("src", GST_STATIC_CAPS ("video/x-raw(ANY)") ); -static GstStaticPadTemplate sink_factory = GST_STATIC_PAD_TEMPLATE ("sink_%u", - GST_PAD_SINK, - GST_PAD_REQUEST, - GST_STATIC_CAPS ("video/x-raw(ANY)") - ); - static void gst_gl_mixer_bin_set_property (GObject * object, guint prop_id, const GValue * value, GParamSpec * pspec); static void gst_gl_mixer_bin_get_property (GObject * object, guint prop_id, @@ -133,6 +127,7 @@ gst_gl_mixer_bin_class_init (GstGLMixerBinClass * klass) { GObjectClass *gobject_class = (GObjectClass *) klass; GstElementClass *element_class = GST_ELEMENT_CLASS (klass); + GstCaps *upload_caps; g_type_class_add_private (klass, sizeof (GstGLMixerBinPrivate)); @@ -178,8 +173,12 @@ gst_gl_mixer_bin_class_init (GstGLMixerBinClass * klass) gst_element_class_add_pad_template (element_class, gst_static_pad_template_get (&src_factory)); + + upload_caps = gst_gl_upload_get_input_template_caps (); gst_element_class_add_pad_template (element_class, - gst_static_pad_template_get (&sink_factory)); + gst_pad_template_new ("sink_%u", GST_PAD_SINK, GST_PAD_REQUEST, + upload_caps)); + gst_caps_unref (upload_caps); } static void From a9fc9344a34e09e2bc76dd87d1181b3abe479005 Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Wed, 29 Apr 2015 22:55:00 +1000 Subject: [PATCH 159/381] Revert "glvideomixer: implement with glmixerbin" This reverts commit 0fb56738a14391f248aa0be8756adeaf978baa0c. --- ext/gl/gstglvideomixer.c | 313 +++++++-------------------------------- ext/gl/gstglvideomixer.h | 1 - 2 files changed, 51 insertions(+), 263 deletions(-) diff --git a/ext/gl/gstglvideomixer.c b/ext/gl/gstglvideomixer.c index cbba72a5d7..b519995cdd 100644 --- a/ext/gl/gstglvideomixer.c +++ b/ext/gl/gstglvideomixer.c @@ -44,273 +44,16 @@ #endif #include "gstglvideomixer.h" -#include "gstglmixerbin.h" #define GST_CAT_DEFAULT gst_gl_video_mixer_debug GST_DEBUG_CATEGORY_STATIC (GST_CAT_DEFAULT); -#define GST_GL_TYPE_VIDEO_MIXER_BACKGROUND (gst_gl_video_mixer_background_get_type()) -static GType -gst_gl_video_mixer_background_get_type (void) -{ - static GType mixer_background_type = 0; - - static const GEnumValue mixer_background[] = { - {GST_GL_VIDEO_MIXER_BACKGROUND_CHECKER, "Checker pattern", "checker"}, - {GST_GL_VIDEO_MIXER_BACKGROUND_BLACK, "Black", "black"}, - {GST_GL_VIDEO_MIXER_BACKGROUND_WHITE, "White", "white"}, - {GST_GL_VIDEO_MIXER_BACKGROUND_TRANSPARENT, - "Transparent Background to enable further compositing", "transparent"}, - {0, NULL, NULL}, - }; - - if (!mixer_background_type) { - mixer_background_type = - g_enum_register_static ("GstGLVideoMixerBackground", mixer_background); - } - return mixer_background_type; -} - -#define DEFAULT_PAD_XPOS 0 -#define DEFAULT_PAD_YPOS 0 -#define DEFAULT_PAD_WIDTH 0 -#define DEFAULT_PAD_HEIGHT 0 -#define DEFAULT_PAD_ALPHA 1.0 -#define DEFAULT_PAD_ZORDER 0 - -enum -{ - PROP_INPUT_0, - PROP_INPUT_XPOS, - PROP_INPUT_YPOS, - PROP_INPUT_WIDTH, - PROP_INPUT_HEIGHT, - PROP_INPUT_ALPHA, - PROP_INPUT_ZORDER, -}; - -static void gst_gl_video_mixer_input_get_property (GObject * object, - guint prop_id, GValue * value, GParamSpec * pspec); -static void gst_gl_video_mixer_input_set_property (GObject * object, - guint prop_id, const GValue * value, GParamSpec * pspec); - -static GstFlowReturn gst_gl_video_mixer_input_chain (GstPad * pad, - GstObject * parent, GstBuffer * buffer); -static GstFlowReturn gst_gl_video_mixer_input_event (GstPad * pad, - GstObject * parent, GstEvent * event); - -typedef struct _GstGLVideoMixerInput GstGLVideoMixerInput; -typedef GstGhostPadClass GstGLVideoMixerInputClass; - -struct _GstGLVideoMixerInput -{ - GstGhostPad parent; - - GstSegment segment; - - GstPad *mixer_pad; -}; - -GType gst_gl_video_mixer_input_get_type (void); -G_DEFINE_TYPE (GstGLVideoMixerInput, gst_gl_video_mixer_input, - GST_TYPE_GHOST_PAD); - -static void -gst_gl_video_mixer_input_init (GstGLVideoMixerInput * self) -{ - GstPad *pad = GST_PAD (self); - - gst_pad_set_event_function (pad, gst_gl_video_mixer_input_event); -} - -static void -gst_gl_video_mixer_input_class_init (GstGLVideoMixerInputClass * klass) -{ - GObjectClass *gobject_class = G_OBJECT_CLASS (klass); - - gobject_class->set_property = gst_gl_video_mixer_input_set_property; - gobject_class->get_property = gst_gl_video_mixer_input_get_property; - - g_object_class_install_property (gobject_class, PROP_INPUT_ZORDER, - g_param_spec_uint ("zorder", "Z-Order", "Z Order of the picture", - 0, 10000, DEFAULT_PAD_ZORDER, - G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); - g_object_class_install_property (gobject_class, PROP_INPUT_XPOS, - g_param_spec_int ("xpos", "X Position", "X Position of the picture", - G_MININT, G_MAXINT, DEFAULT_PAD_XPOS, - G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); - g_object_class_install_property (gobject_class, PROP_INPUT_YPOS, - g_param_spec_int ("ypos", "Y Position", "Y Position of the picture", - G_MININT, G_MAXINT, DEFAULT_PAD_YPOS, - G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); - g_object_class_install_property (gobject_class, PROP_INPUT_WIDTH, - g_param_spec_int ("width", "Width", "Width of the picture", - G_MININT, G_MAXINT, DEFAULT_PAD_WIDTH, - G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); - g_object_class_install_property (gobject_class, PROP_INPUT_HEIGHT, - g_param_spec_int ("height", "Height", "Height of the picture", - G_MININT, G_MAXINT, DEFAULT_PAD_HEIGHT, - G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); - g_object_class_install_property (gobject_class, PROP_INPUT_ALPHA, - g_param_spec_double ("alpha", "Alpha", "Alpha of the picture", 0.0, 1.0, - DEFAULT_PAD_ALPHA, - G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); -} - -static void -gst_gl_video_mixer_input_get_property (GObject * object, guint prop_id, - GValue * value, GParamSpec * pspec) -{ - GstGLVideoMixerInput *self = (GstGLVideoMixerInput *) object; - - if (self->mixer_pad) - g_object_get_property (G_OBJECT (self->mixer_pad), pspec->name, value); -} - -static void -gst_gl_video_mixer_input_set_property (GObject * object, guint prop_id, - const GValue * value, GParamSpec * pspec) -{ - GstGLVideoMixerInput *self = (GstGLVideoMixerInput *) object; - - if (self->mixer_pad) - g_object_set_property (G_OBJECT (self->mixer_pad), pspec->name, value); -} - -static GstFlowReturn -gst_gl_video_mixer_input_chain (GstPad * pad, GstObject * parent, - GstBuffer * buffer) -{ - GstGLVideoMixerInput *self = (GstGLVideoMixerInput *) pad; - GstClockTime timestamp, stream_time; -// gdouble alpha; - - timestamp = GST_BUFFER_TIMESTAMP (buffer); - - stream_time = - gst_segment_to_stream_time (&self->segment, GST_FORMAT_TIME, timestamp); - - gst_object_sync_values (GST_OBJECT (self), stream_time); -#if 0 - /* FIXME: implement no-upload on alpha = 0 */ - g_object_get (self, "alpha", &alpha, NULL); - - if (alpha <= 0.0) { - GST_DEBUG_OBJECT (self, "dropping buffer %" GST_PTR_FORMAT - " due to alpha value %f", buffer, alpha); - gst_buffer_unref (buffer); - return GST_FLOW_OK; - } -#endif - return gst_proxy_pad_chain_default (pad, parent, buffer); -} - -static GstFlowReturn -gst_gl_video_mixer_input_event (GstPad * pad, GstObject * parent, - GstEvent * event) -{ - GstGLVideoMixerInput *self = (GstGLVideoMixerInput *) pad; - - switch (GST_EVENT_TYPE (event)) { - case GST_EVENT_SEGMENT: - gst_event_copy_segment (event, &self->segment); - break; - default: - break; - } - - return gst_pad_event_default (pad, parent, event); -} - -static GstGhostPad * -_create_video_mixer_input (GstGLMixerBin * self, GstPad * mixer_pad) -{ - GstGLVideoMixerInput *input = - g_object_new (gst_gl_video_mixer_input_get_type (), "name", - GST_OBJECT_NAME (mixer_pad), "direction", GST_PAD_DIRECTION (mixer_pad), - NULL); - - if (!gst_ghost_pad_construct (GST_GHOST_PAD (input))) { - gst_object_unref (input); - return NULL; - } - - gst_pad_set_chain_function (GST_PAD (input), gst_gl_video_mixer_input_chain); - - input->mixer_pad = mixer_pad; - - return GST_GHOST_PAD (input); -} - -enum -{ - PROP_BIN_0, - PROP_BIN_BACKGROUND, -}; -#define DEFAULT_BACKGROUND GST_GL_VIDEO_MIXER_BACKGROUND_CHECKER - -static void gst_gl_video_mixer_bin_get_property (GObject * object, - guint prop_id, GValue * value, GParamSpec * pspec); -static void gst_gl_video_mixer_bin_set_property (GObject * object, - guint prop_id, const GValue * value, GParamSpec * pspec); - -typedef GstGLMixerBin GstGLVideoMixerBin; -typedef GstGLMixerBinClass GstGLVideoMixerBinClass; - -G_DEFINE_TYPE (GstGLVideoMixerBin, gst_gl_video_mixer_bin, - GST_TYPE_GL_MIXER_BIN); - -static void -gst_gl_video_mixer_bin_init (GstGLVideoMixerBin * self) -{ - GstGLMixerBin *mix_bin = GST_GL_MIXER_BIN (self); - - gst_gl_mixer_bin_finish_init_with_element (mix_bin, - g_object_new (GST_TYPE_GL_VIDEO_MIXER, NULL)); -} - -static void -gst_gl_video_mixer_bin_class_init (GstGLVideoMixerBinClass * klass) -{ - GstGLMixerBinClass *mixer_class = GST_GL_MIXER_BIN_CLASS (klass); - GObjectClass *gobject_class = G_OBJECT_CLASS (klass); - - mixer_class->create_input_pad = _create_video_mixer_input; - - gobject_class->set_property = gst_gl_video_mixer_bin_set_property; - gobject_class->get_property = gst_gl_video_mixer_bin_get_property; - - g_object_class_install_property (gobject_class, PROP_BIN_BACKGROUND, - g_param_spec_enum ("background", "Background", "Background type", - GST_GL_TYPE_VIDEO_MIXER_BACKGROUND, - DEFAULT_BACKGROUND, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); -} - -static void -gst_gl_video_mixer_bin_get_property (GObject * object, guint prop_id, - GValue * value, GParamSpec * pspec) -{ - GstGLMixerBin *self = GST_GL_MIXER_BIN (object); - - if (self->mixer) - g_object_get_property (G_OBJECT (self->mixer), pspec->name, value); -} - -static void -gst_gl_video_mixer_bin_set_property (GObject * object, guint prop_id, - const GValue * value, GParamSpec * pspec) -{ - GstGLMixerBin *self = GST_GL_MIXER_BIN (object); - - if (self->mixer) - g_object_set_property (G_OBJECT (self->mixer), pspec->name, value); -} - enum { PROP_0, PROP_BACKGROUND, }; +#define DEFAULT_BACKGROUND GST_GL_VIDEO_MIXER_BACKGROUND_CHECKER #define DEBUG_INIT \ GST_DEBUG_CATEGORY_INIT (gst_gl_video_mixer_debug, "glvideomixer", 0, "glvideomixer element"); @@ -435,7 +178,14 @@ static void gst_gl_video_mixer_pad_set_property (GObject * object, guint prop_id, const GValue * value, GParamSpec * pspec); static void gst_gl_video_mixer_pad_get_property (GObject * object, guint prop_id, GValue * value, GParamSpec * pspec); +static GstBuffer *gst_gl_video_mixer_pad_upload_buffer (GstGLMixer * mix, + GstGLMixerFrameData * frame, GstBuffer * buffer); +#define DEFAULT_PAD_XPOS 0 +#define DEFAULT_PAD_YPOS 0 +#define DEFAULT_PAD_WIDTH 0 +#define DEFAULT_PAD_HEIGHT 0 +#define DEFAULT_PAD_ALPHA 1.0 enum { PROP_PAD_0, @@ -456,6 +206,7 @@ static void gst_gl_video_mixer_pad_class_init (GstGLVideoMixerPadClass * klass) { GObjectClass *gobject_class = (GObjectClass *) klass; + GstGLMixerPadClass *mix_pad_class = (GstGLMixerPadClass *) klass; gobject_class->set_property = gst_gl_video_mixer_pad_set_property; gobject_class->get_property = gst_gl_video_mixer_pad_get_property; @@ -480,6 +231,8 @@ gst_gl_video_mixer_pad_class_init (GstGLVideoMixerPadClass * klass) g_param_spec_double ("alpha", "Alpha", "Alpha of the picture", 0.0, 1.0, DEFAULT_PAD_ALPHA, G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); + + mix_pad_class->upload_buffer = gst_gl_video_mixer_pad_upload_buffer; } static void @@ -545,6 +298,44 @@ gst_gl_video_mixer_pad_set_property (GObject * object, guint prop_id, gst_object_unref (mix); } +static GstBuffer * +gst_gl_video_mixer_pad_upload_buffer (GstGLMixer * mix, + GstGLMixerFrameData * frame, GstBuffer * buffer) +{ + GstGLVideoMixerPad *pad = GST_GL_VIDEO_MIXER_PAD (frame->pad); + + if (pad->alpha > 0.0) { + return + GST_GL_MIXER_PAD_CLASS + (gst_gl_video_mixer_pad_parent_class)->upload_buffer (mix, frame, + buffer); + } + + return NULL; +} + +#define GST_GL_TYPE_VIDEO_MIXER_BACKGROUND (gst_gl_video_mixer_background_get_type()) +static GType +gst_gl_video_mixer_background_get_type (void) +{ + static GType mixer_background_type = 0; + + static const GEnumValue mixer_background[] = { + {GST_GL_VIDEO_MIXER_BACKGROUND_CHECKER, "Checker pattern", "checker"}, + {GST_GL_VIDEO_MIXER_BACKGROUND_BLACK, "Black", "black"}, + {GST_GL_VIDEO_MIXER_BACKGROUND_WHITE, "White", "white"}, + {GST_GL_VIDEO_MIXER_BACKGROUND_TRANSPARENT, + "Transparent Background to enable further compositing", "transparent"}, + {0, NULL, NULL}, + }; + + if (!mixer_background_type) { + mixer_background_type = + g_enum_register_static ("GstGLVideoMixerBackground", mixer_background); + } + return mixer_background_type; +} + static void gst_gl_video_mixer_class_init (GstGLVideoMixerClass * klass) { @@ -561,7 +352,7 @@ gst_gl_video_mixer_class_init (GstGLVideoMixerClass * klass) gst_element_class_set_metadata (element_class, "OpenGL video_mixer", "Filter/Effect/Video/Compositor", "OpenGL video_mixer", - "Matthew Waters "); + "Julien Isorce "); g_object_class_install_property (gobject_class, PROP_BACKGROUND, g_param_spec_enum ("background", "Background", "Background type", @@ -663,9 +454,7 @@ _update_caps (GstVideoAggregator * vagg, GstCaps * caps) } GST_OBJECT_UNLOCK (vagg); - ret = - GST_VIDEO_AGGREGATOR_CLASS (gst_gl_video_mixer_parent_class)->update_caps - (vagg, caps); + ret = gst_gl_mixer_update_caps (GST_GL_MIXER (vagg), caps); for (i = 0; i < gst_caps_get_size (ret); i++) { GstStructure *s = gst_caps_get_structure (ret, i); diff --git a/ext/gl/gstglvideomixer.h b/ext/gl/gstglvideomixer.h index 967358baba..b351888fce 100644 --- a/ext/gl/gstglvideomixer.h +++ b/ext/gl/gstglvideomixer.h @@ -73,7 +73,6 @@ struct _GstGLVideoMixerClass }; GType gst_gl_video_mixer_get_type (void); -GType gst_gl_video_mixer_bin_get_type (void); G_END_DECLS From 2c7c82d748c57e15bb307a4bfbe437c7f18b41f7 Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Thu, 30 Apr 2015 11:15:40 +1000 Subject: [PATCH 160/381] gl: readd glupload/download onto element pads Allows insertion of gl elements into non-gl pipelines without converter (upload/download) elements. https://bugzilla.gnome.org/show_bug.cgi?id=743974 --- ext/gl/gstglmixer.c | 178 ++++++++++++++++++++++++++++++++++--------- ext/gl/gstglmixer.h | 8 ++ ext/gl/gstglmosaic.c | 3 + 3 files changed, 154 insertions(+), 35 deletions(-) diff --git a/ext/gl/gstglmixer.c b/ext/gl/gstglmixer.c index 8e35af10b0..907a160dc4 100644 --- a/ext/gl/gstglmixer.c +++ b/ext/gl/gstglmixer.c @@ -42,6 +42,9 @@ static void gst_gl_mixer_pad_get_property (GObject * object, guint prop_id, GValue * value, GParamSpec * pspec); static void gst_gl_mixer_pad_set_property (GObject * object, guint prop_id, const GValue * value, GParamSpec * pspec); +static void gst_gl_mixer_pad_finalize (GObject * object); +static GstBuffer *_default_pad_upload_buffer (GstGLMixer * mix, + GstGLMixerFrameData * frame, GstBuffer * buffer); enum { @@ -71,10 +74,13 @@ gst_gl_mixer_pad_class_init (GstGLMixerPadClass * klass) gobject_class->set_property = gst_gl_mixer_pad_set_property; gobject_class->get_property = gst_gl_mixer_pad_get_property; + gobject_class->finalize = gst_gl_mixer_pad_finalize; vaggpad_class->set_info = NULL; vaggpad_class->prepare_frame = NULL; vaggpad_class->clean_frame = NULL; + + klass->upload_buffer = _default_pad_upload_buffer; } static void @@ -198,12 +204,22 @@ gst_gl_mixer_pad_sink_acceptcaps (GstPad * pad, GstGLMixer * mix, return ret; } -/* copies the given caps */ -static GstCaps * -_update_caps (GstVideoAggregator * vagg, GstCaps * caps) +GstCaps * +gst_gl_mixer_update_caps (GstGLMixer * mix, GstCaps * caps) { - return gst_gl_caps_replace_all_caps_features (caps, - GST_CAPS_FEATURE_MEMORY_GL_MEMORY); + GstGLContext *context = GST_GL_BASE_MIXER (mix)->context; + GstCaps *result, *tmp; + + result = + gst_caps_from_string (GST_VIDEO_CAPS_MAKE_WITH_FEATURES + (GST_CAPS_FEATURE_MEMORY_GL_MEMORY, "RGBA")); + + tmp = gst_gl_upload_transform_caps (context, GST_PAD_SRC, result, NULL); + gst_caps_unref (result); + result = tmp; + GST_DEBUG_OBJECT (mix, "transfer returned caps %" GST_PTR_FORMAT, result); + + return result; } static GstCaps * @@ -284,6 +300,17 @@ gst_gl_mixer_pad_init (GstGLMixerPad * mixerpad) { } +static void +gst_gl_mixer_pad_finalize (GObject * object) +{ + GstGLMixerPad *pad = GST_GL_MIXER_PAD (object); + + gst_buffer_replace (&pad->uploaded_buffer, NULL); + gst_object_replace ((GstObject **) & pad->upload, NULL); + + G_OBJECT_CLASS (gst_gl_mixer_pad_parent_class)->finalize (object); +} + /* GLMixer signals and args */ enum { @@ -301,15 +328,7 @@ static GstStaticPadTemplate src_factory = GST_STATIC_PAD_TEMPLATE ("src", GST_PAD_ALWAYS, GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES (GST_CAPS_FEATURE_MEMORY_GL_MEMORY, - "RGBA")) - ); - -static GstStaticPadTemplate sink_factory = GST_STATIC_PAD_TEMPLATE ("sink_%u", - GST_PAD_SINK, - GST_PAD_REQUEST, - GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES - (GST_CAPS_FEATURE_MEMORY_GL_MEMORY, - "RGBA")) + "RGBA") ";" GST_VIDEO_CAPS_MAKE ("RGBA")) ); static gboolean gst_gl_mixer_src_query (GstAggregator * agg, GstQuery * query); @@ -340,7 +359,8 @@ gst_gl_mixer_class_init (GstGLMixerClass * klass) GstVideoAggregatorClass *videoaggregator_class = (GstVideoAggregatorClass *) klass; GstAggregatorClass *agg_class = (GstAggregatorClass *) klass; - GstGLBaseMixerClass *mix_class = GST_GL_BASE_MIXER_CLASS (klass);; + GstGLBaseMixerClass *mix_class = GST_GL_BASE_MIXER_CLASS (klass); + GstCaps *upload_caps, *elem_caps, *templ; GST_DEBUG_CATEGORY_INIT (GST_CAT_DEFAULT, "glmixer", 0, "OpenGL mixer"); @@ -353,8 +373,15 @@ gst_gl_mixer_class_init (GstGLMixerClass * klass) gst_element_class_add_pad_template (element_class, gst_static_pad_template_get (&src_factory)); + + elem_caps = gst_caps_from_string ("video/x-raw(ANY),format=RGBA"); + upload_caps = gst_gl_upload_get_input_template_caps (); + templ = gst_caps_intersect (elem_caps, upload_caps); gst_element_class_add_pad_template (element_class, - gst_static_pad_template_get (&sink_factory)); + gst_pad_template_new ("sink_%u", GST_PAD_SINK, GST_PAD_REQUEST, templ)); + gst_caps_unref (upload_caps); + gst_caps_unref (elem_caps); + gst_caps_unref (templ); agg_class->sinkpads_type = GST_TYPE_GL_MIXER_PAD; agg_class->sink_query = gst_gl_mixer_sink_query; @@ -365,7 +392,6 @@ gst_gl_mixer_class_init (GstGLMixerClass * klass) videoaggregator_class->aggregate_frames = gst_gl_mixer_aggregate_frames; videoaggregator_class->get_output_buffer = gst_gl_mixer_get_output_buffer; videoaggregator_class->negotiated_caps = _negotiated_caps; - videoaggregator_class->update_caps = _update_caps; videoaggregator_class->find_best_format = NULL; mix_class->propose_allocation = gst_gl_mixer_propose_allocation; @@ -545,8 +571,11 @@ gst_gl_mixer_decide_allocation (GstGLBaseMixer * base_mix, GstQuery * query) update_pool = FALSE; } - if (!pool) + if (!pool || !GST_IS_GL_BUFFER_POOL (pool)) { + if (pool) + gst_object_unref (pool); pool = gst_gl_buffer_pool_new (context); + } config = gst_buffer_pool_get_config (pool); gst_buffer_pool_config_set_params (config, caps, size, min, max); @@ -570,6 +599,63 @@ context_error: } } +static GstBuffer * +_default_pad_upload_buffer (GstGLMixer * mix, GstGLMixerFrameData * frame, + GstBuffer * buffer) +{ + GstVideoAggregatorPad *vaggpad = GST_VIDEO_AGGREGATOR_PAD (frame->pad); + GstGLMixerPad *pad = frame->pad; + GstVideoInfo gl_info; + GstGLSyncMeta *sync_meta; + + gst_video_info_set_format (&gl_info, + GST_VIDEO_FORMAT_RGBA, + GST_VIDEO_INFO_WIDTH (&vaggpad->info), + GST_VIDEO_INFO_HEIGHT (&vaggpad->info)); + + if (!pad->upload) { + GstCaps *in_caps, *upload_caps; + + in_caps = gst_pad_get_current_caps (GST_PAD (pad)); + upload_caps = gst_video_info_to_caps (&gl_info); + gst_caps_set_features (upload_caps, 0, + gst_caps_features_from_string (GST_CAPS_FEATURE_MEMORY_GL_MEMORY)); + + pad->upload = gst_gl_upload_new (GST_GL_BASE_MIXER (mix)->context); + + if (!gst_gl_upload_set_caps (pad->upload, in_caps, upload_caps)) { + GST_WARNING_OBJECT (pad, "Failed to configure uploader"); + gst_caps_unref (in_caps); + gst_caps_unref (upload_caps); + gst_object_unref (pad->upload); + pad->upload = NULL; + return NULL; + } + + gst_caps_unref (in_caps); + gst_caps_unref (upload_caps); + } + + sync_meta = gst_buffer_get_gl_sync_meta (vaggpad->buffer); + if (sync_meta) + gst_gl_sync_meta_wait (sync_meta, GST_GL_BASE_MIXER (mix)->context); + + if (GST_GL_UPLOAD_DONE != gst_gl_upload_perform_with_buffer (pad->upload, + vaggpad->buffer, &pad->uploaded_buffer) || !pad->uploaded_buffer) { + GST_WARNING_OBJECT (pad, "Failed to upload input buffer %p", + vaggpad->buffer); + return NULL; + } + + if (gst_video_frame_map (&frame->v_frame, &gl_info, pad->uploaded_buffer, + GST_MAP_READ | GST_MAP_GL)) { + frame->texture = *(guint *) frame->v_frame.data[0]; + frame->mapped = TRUE; + } + + return pad->uploaded_buffer; +} + gboolean gst_gl_mixer_process_textures (GstGLMixer * mix, GstBuffer * outbuf) { @@ -577,7 +663,7 @@ gst_gl_mixer_process_textures (GstGLMixer * mix, GstBuffer * outbuf) GList *walk; guint out_tex; gboolean res = TRUE; - guint array_index = 0; + gint array_index = 0; GstVideoFrame out_frame; GstElement *element = GST_ELEMENT (mix); GstVideoAggregator *vagg = GST_VIDEO_AGGREGATOR (mix); @@ -602,34 +688,28 @@ gst_gl_mixer_process_textures (GstGLMixer * mix, GstBuffer * outbuf) mix->frames->pdata[i] = g_slice_new0 (GstGLMixerFrameData); while (walk) { GstGLMixerPad *pad = GST_GL_MIXER_PAD (walk->data); + GstGLMixerPadClass *pad_class = GST_GL_MIXER_PAD_GET_CLASS (pad); GstVideoAggregatorPad *vaggpad = walk->data; GstGLMixerFrameData *frame; frame = g_ptr_array_index (mix->frames, array_index); frame->pad = pad; frame->texture = 0; + frame->mapped = FALSE; walk = g_list_next (walk); if (vaggpad->buffer != NULL) { - GstVideoInfo gl_info; - GstVideoFrame gl_frame; - GstGLSyncMeta *sync_meta; + g_assert (pad_class->upload_buffer); - gst_video_info_set_format (&gl_info, - GST_VIDEO_FORMAT_RGBA, - GST_VIDEO_INFO_WIDTH (&vaggpad->info), - GST_VIDEO_INFO_HEIGHT (&vaggpad->info)); + if (pad->uploaded_buffer) + gst_buffer_unref (pad->uploaded_buffer); + pad->uploaded_buffer = + pad_class->upload_buffer (mix, frame, vaggpad->buffer); - sync_meta = gst_buffer_get_gl_sync_meta (vaggpad->buffer); - if (sync_meta) - gst_gl_sync_meta_wait (sync_meta, GST_GL_BASE_MIXER (mix)->context); - - if (gst_video_frame_map (&gl_frame, &gl_info, vaggpad->buffer, - GST_MAP_READ | GST_MAP_GL)) { - frame->texture = *(guint *) gl_frame.data[0]; - gst_video_frame_unmap (&gl_frame); - } + GST_DEBUG_OBJECT (pad, + "uploaded buffer %" GST_PTR_FORMAT " from buffer %" GST_PTR_FORMAT, + pad->uploaded_buffer, vaggpad->buffer); } ++array_index; @@ -652,6 +732,19 @@ gst_gl_mixer_process_textures (GstGLMixer * mix, GstBuffer * outbuf) g_mutex_unlock (&priv->gl_resource_lock); out: + for (--array_index; array_index >= 0; array_index--) { + GstGLMixerFrameData *frame; + + frame = g_ptr_array_index (mix->frames, array_index); + + if (frame->mapped) + gst_video_frame_unmap (&frame->v_frame); + frame->mapped = FALSE; + frame->texture = 0; + + gst_buffer_replace (&frame->pad->uploaded_buffer, NULL); + } + GST_OBJECT_UNLOCK (mix); gst_video_frame_unmap (&out_frame); @@ -761,6 +854,18 @@ gst_gl_mixer_start (GstAggregator * agg) return GST_AGGREGATOR_CLASS (parent_class)->start (agg); } +static gboolean +_clean_upload (GstAggregator * agg, GstAggregatorPad * aggpad, + gpointer user_data) +{ + GstGLMixerPad *pad = GST_GL_MIXER_PAD (aggpad); + + gst_buffer_replace (&pad->uploaded_buffer, NULL); + gst_object_replace ((GstObject **) & pad->upload, NULL); + + return TRUE; +} + static gboolean gst_gl_mixer_stop (GstAggregator * agg) { @@ -768,6 +873,9 @@ gst_gl_mixer_stop (GstAggregator * agg) GstGLMixerClass *mixer_class = GST_GL_MIXER_GET_CLASS (mix); GstGLContext *context = GST_GL_BASE_MIXER (mix)->context; + gst_aggregator_iterate_sinkpads (agg, + (GstAggregatorPadForeachFunc) _clean_upload, NULL); + GST_OBJECT_LOCK (agg); g_ptr_array_free (mix->frames, TRUE); mix->frames = NULL; diff --git a/ext/gl/gstglmixer.h b/ext/gl/gstglmixer.h index 01eed34f20..38b51d369d 100644 --- a/ext/gl/gstglmixer.h +++ b/ext/gl/gstglmixer.h @@ -52,11 +52,16 @@ typedef struct _GstGLMixerPadClass GstGLMixerPadClass; struct _GstGLMixerPad { GstGLBaseMixerPad parent; + + GstGLUpload *upload; + GstBuffer *uploaded_buffer; }; struct _GstGLMixerPadClass { GstGLBaseMixerPadClass parent_class; + + GstBuffer * (*upload_buffer) (GstGLMixer * mix, GstGLMixerFrameData * frame, GstBuffer * buffer); }; GType gst_gl_mixer_pad_get_type (void); @@ -109,12 +114,15 @@ struct _GstGLMixerClass struct _GstGLMixerFrameData { GstGLMixerPad *pad; + gboolean mapped; + GstVideoFrame v_frame; guint texture; }; GType gst_gl_mixer_get_type(void); gboolean gst_gl_mixer_process_textures (GstGLMixer * mix, GstBuffer * outbuf); +GstCaps * gst_gl_mixer_update_caps (GstGLMixer * mix, GstCaps * caps); G_END_DECLS #endif /* __GST_GL_MIXER_H__ */ diff --git a/ext/gl/gstglmosaic.c b/ext/gl/gstglmosaic.c index 726674b4df..0dc13b7c67 100644 --- a/ext/gl/gstglmosaic.c +++ b/ext/gl/gstglmosaic.c @@ -190,6 +190,9 @@ gst_gl_mosaic_init_shader (GstGLMixer * mixer, GstCaps * outcaps) { GstGLMosaic *mosaic = GST_GL_MOSAIC (mixer); + if (mosaic->shader) + gst_object_unref (mosaic->shader); + //blocking call, wait the opengl thread has compiled the shader return gst_gl_context_gen_shader (GST_GL_BASE_MIXER (mixer)->context, mosaic_v_src, mosaic_f_src, &mosaic->shader); From 8f1edb69f4fc4b6b0c098f0c9d11a6f32bdd738b Mon Sep 17 00:00:00 2001 From: Nirbheek Chauhan Date: Sat, 18 Apr 2015 15:09:02 +0530 Subject: [PATCH 161/381] compositor: Skip pads that are completely obscured by a higher zorder pad For each frame, compare the frame boundaries, check if the format contains an alpha channel, check opacity, and skip the frame if it's going to be completely overwritten by a higher zorder frame. The check is O(n^2), but that doesn't matter here because the number of sinkpads is small. More can be done to avoid needless drawing, but this covers the majority of cases. See TODOs. Ideally, a reverse painter's algorithm should be used for optimal drawing, but memcpy during compositing is small compared to the CPU used for frame conversion on each pad. https://bugzilla.gnome.org/show_bug.cgi?id=746147 --- gst/compositor/compositor.c | 84 +++++++++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) diff --git a/gst/compositor/compositor.c b/gst/compositor/compositor.c index ef71d841b3..357dd4b6ce 100644 --- a/gst/compositor/compositor.c +++ b/gst/compositor/compositor.c @@ -296,6 +296,17 @@ gst_compositor_pad_set_info (GstVideoAggregatorPad * pad, return TRUE; } +/* Test whether rectangle2 contains rectangle 1 (geometrically) */ +static gboolean +is_rectangle_contained (GstVideoRectangle rect1, GstVideoRectangle rect2) +{ + if ((rect2.x <= rect1.x) && (rect2.y <= rect1.y) && + ((rect2.x + rect2.w) >= (rect1.x + rect1.w)) && + ((rect2.y + rect2.h) >= (rect1.y + rect1.h))) + return TRUE; + return FALSE; +} + static gboolean gst_compositor_pad_prepare_frame (GstVideoAggregatorPad * pad, GstVideoAggregator * vagg) @@ -307,6 +318,11 @@ gst_compositor_pad_prepare_frame (GstVideoAggregatorPad * pad, GstVideoFrame *frame; static GstAllocationParams params = { 0, 15, 0, 0, }; gint width, height; + gboolean frame_obscured = FALSE; + GList *l; + /* The rectangle representing this frame, clamped to the video's boundaries. + * Due to the clamping, this is different from the frame width/height above. */ + GstVideoRectangle frame_rect; if (!pad->buffer) return TRUE; @@ -319,6 +335,17 @@ gst_compositor_pad_prepare_frame (GstVideoAggregatorPad * pad, return FALSE; } + /* There's three types of width/height here: + * 1. GST_VIDEO_FRAME_WIDTH/HEIGHT: + * The frame width/height (same as pad->buffer_vinfo.height/width; + * see gst_video_frame_map()) + * 2. cpad->width/height: + * The optional pad property for scaling the frame (if zero, the video is + * left unscaled) + * 3. conversion_info.width/height: + * Equal to cpad->width/height if it's set, otherwise it's the frame + * width/height. See ->set_info() + * */ if (cpad->width > 0) width = cpad->width; else @@ -394,6 +421,60 @@ gst_compositor_pad_prepare_frame (GstVideoAggregatorPad * pad, g_free (wanted_colorimetry); } + /* Clamp the x/y coordinates of this frame to the video boundaries to cover + * the case where (say, with negative xpos/ypos) the non-obscured portion of + * the frame could be outside the bounds of the video itself and hence not + * visible at all */ + frame_rect.x = CLAMP (cpad->xpos, 0, + GST_VIDEO_INFO_WIDTH (&pad->buffer_vinfo)); + frame_rect.y = CLAMP (cpad->ypos, 0, + GST_VIDEO_INFO_HEIGHT (&pad->buffer_vinfo)); + /* Clamp the width/height to the frame boundaries as well */ + frame_rect.w = MIN (GST_VIDEO_INFO_WIDTH (&cpad->conversion_info), + GST_VIDEO_INFO_WIDTH (&pad->buffer_vinfo) - cpad->xpos); + frame_rect.h = MIN (GST_VIDEO_INFO_HEIGHT (&cpad->conversion_info), + GST_VIDEO_INFO_HEIGHT (&pad->buffer_vinfo) - cpad->ypos); + + GST_OBJECT_LOCK (vagg); + /* Check if this frame is obscured by a higher-zorder frame + * TODO: Also skip a frame if it's obscured by a combination of + * higher-zorder frames */ + for (l = g_list_find (GST_ELEMENT (vagg)->sinkpads, pad)->next; l; + l = l->next) { + GstVideoRectangle frame2_rect; + GstVideoAggregatorPad *pad2 = l->data; + GstCompositorPad *cpad2 = GST_COMPOSITOR_PAD (pad2); + + /* We don't need to clamp the coords of the second rectangle */ + frame2_rect.x = cpad2->xpos; + frame2_rect.y = cpad2->ypos; + /* This is effectively what set_info and the above conversion + * code do to calculate the desired width/height */ + frame2_rect.w = cpad2->width ? cpad2->width : + GST_VIDEO_INFO_WIDTH (&cpad2->conversion_info); + frame2_rect.h = cpad2->height ? cpad2->height : + GST_VIDEO_INFO_HEIGHT (&cpad2->conversion_info); + + /* Check if there's a buffer to be aggregated, ensure it can't have an alpha + * channel, then check opacity and frame boundaries */ + if (pad2->buffer && cpad2->alpha == 1.0 && + !GST_VIDEO_INFO_HAS_ALPHA (&pad2->info) && + is_rectangle_contained (frame_rect, frame2_rect)) { + frame_obscured = TRUE; + GST_DEBUG_OBJECT (pad, "Obscured by %s, skipping frame", + GST_PAD_NAME (pad2)); + break; + } + } + GST_OBJECT_UNLOCK (vagg); + + if (frame_obscured) { + converted_frame = NULL; + gst_video_frame_unmap (frame); + g_slice_free (GstVideoFrame, frame); + goto done; + } + if (cpad->alpha == 0.0) { GST_DEBUG_OBJECT (vagg, "Pad has alpha 0.0, not converting frame"); converted_frame = NULL; @@ -428,6 +509,7 @@ gst_compositor_pad_prepare_frame (GstVideoAggregatorPad * pad, converted_frame = frame; } +done: pad->aggregated_frame = converted_frame; return TRUE; @@ -808,6 +890,8 @@ gst_compositor_aggregate_frames (GstVideoAggregator * vagg, GstBuffer * outbuf) outframe = &out_frame; /* default to blending */ composite = self->blend; + /* TODO: If the frames to be composited completely obscure the background, + * don't bother drawing the background at all. */ switch (self->background) { case COMPOSITOR_BACKGROUND_CHECKER: self->fill_checker (outframe); From 1817025ba0c6556423a0206c667c7ed64f0a730d Mon Sep 17 00:00:00 2001 From: Nirbheek Chauhan Date: Sat, 18 Apr 2015 15:09:02 +0530 Subject: [PATCH 162/381] compositor: use accessor macros for consistency https://bugzilla.gnome.org/show_bug.cgi?id=746147 --- gst/compositor/compositor.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/gst/compositor/compositor.c b/gst/compositor/compositor.c index 357dd4b6ce..eae152ca1d 100644 --- a/gst/compositor/compositor.c +++ b/gst/compositor/compositor.c @@ -358,8 +358,8 @@ gst_compositor_pad_prepare_frame (GstVideoAggregatorPad * pad, /* The only thing that can change here is the width * and height, otherwise set_info would've been called */ - if (cpad->conversion_info.width != width || - cpad->conversion_info.height != height) { + if (GST_VIDEO_INFO_WIDTH (&cpad->conversion_info) != width || + GST_VIDEO_INFO_HEIGHT (&cpad->conversion_info) != height) { gchar *colorimetry, *wanted_colorimetry; const gchar *chroma, *wanted_chroma; @@ -413,8 +413,8 @@ gst_compositor_pad_prepare_frame (GstVideoAggregatorPad * pad, return FALSE; } } else { - cpad->conversion_info.width = width; - cpad->conversion_info.height = height; + GST_VIDEO_INFO_WIDTH (&cpad->conversion_info) = width; + GST_VIDEO_INFO_HEIGHT (&cpad->conversion_info) = height; } g_free (colorimetry); @@ -486,7 +486,7 @@ gst_compositor_pad_prepare_frame (GstVideoAggregatorPad * pad, converted_frame = g_slice_new0 (GstVideoFrame); /* We wait until here to set the conversion infos, in case vagg->info changed */ - converted_size = cpad->conversion_info.size; + converted_size = GST_VIDEO_INFO_SIZE (&cpad->conversion_info); outsize = GST_VIDEO_INFO_SIZE (&vagg->info); converted_size = converted_size > outsize ? converted_size : outsize; converted_buf = gst_buffer_new_allocate (NULL, converted_size, ¶ms); From 070777498f094ad24846ed7c2c906176de5cdba7 Mon Sep 17 00:00:00 2001 From: Nirbheek Chauhan Date: Sat, 18 Apr 2015 15:10:00 +0530 Subject: [PATCH 163/381] compositor: Only map the frame from a buffer if it will be used It's a waste of resources to map it if it won't be converted or used at all. Since we moved the frame mapping down, we need to use the GST_VIDEO_INFO accessor macros now in the code above that instead of the GST_VIDEO_FRAME accessor macros. https://bugzilla.gnome.org/show_bug.cgi?id=746147 --- gst/compositor/compositor.c | 49 ++++++++++++++++++------------------- 1 file changed, 24 insertions(+), 25 deletions(-) diff --git a/gst/compositor/compositor.c b/gst/compositor/compositor.c index eae152ca1d..756df1ac56 100644 --- a/gst/compositor/compositor.c +++ b/gst/compositor/compositor.c @@ -327,14 +327,6 @@ gst_compositor_pad_prepare_frame (GstVideoAggregatorPad * pad, if (!pad->buffer) return TRUE; - frame = g_slice_new0 (GstVideoFrame); - - if (!gst_video_frame_map (frame, &pad->buffer_vinfo, pad->buffer, - GST_MAP_READ)) { - GST_WARNING_OBJECT (vagg, "Could not map input buffer"); - return FALSE; - } - /* There's three types of width/height here: * 1. GST_VIDEO_FRAME_WIDTH/HEIGHT: * The frame width/height (same as pad->buffer_vinfo.height/width; @@ -343,18 +335,18 @@ gst_compositor_pad_prepare_frame (GstVideoAggregatorPad * pad, * The optional pad property for scaling the frame (if zero, the video is * left unscaled) * 3. conversion_info.width/height: - * Equal to cpad->width/height if it's set, otherwise it's the frame + * Equal to cpad->width/height if it's set, otherwise it's the pad * width/height. See ->set_info() * */ if (cpad->width > 0) width = cpad->width; else - width = GST_VIDEO_FRAME_WIDTH (frame); + width = GST_VIDEO_INFO_WIDTH (&pad->buffer_vinfo); if (cpad->height > 0) height = cpad->height; else - height = GST_VIDEO_FRAME_HEIGHT (frame); + height = GST_VIDEO_INFO_HEIGHT (&pad->buffer_vinfo); /* The only thing that can change here is the width * and height, otherwise set_info would've been called */ @@ -371,20 +363,21 @@ gst_compositor_pad_prepare_frame (GstVideoAggregatorPad * pad, gst_video_converter_free (cpad->convert); cpad->convert = NULL; - colorimetry = gst_video_colorimetry_to_string (&frame->info.colorimetry); - chroma = gst_video_chroma_to_string (frame->info.chroma_site); + colorimetry = + gst_video_colorimetry_to_string (&pad->buffer_vinfo.colorimetry); + chroma = gst_video_chroma_to_string (pad->buffer_vinfo.chroma_site); wanted_colorimetry = gst_video_colorimetry_to_string (&cpad->conversion_info.colorimetry); wanted_chroma = gst_video_chroma_to_string (cpad->conversion_info.chroma_site); - if (GST_VIDEO_INFO_FORMAT (&frame->info) != + if (GST_VIDEO_INFO_FORMAT (&pad->buffer_vinfo) != GST_VIDEO_INFO_FORMAT (&cpad->conversion_info) || g_strcmp0 (colorimetry, wanted_colorimetry) || g_strcmp0 (chroma, wanted_chroma) - || width != GST_VIDEO_FRAME_WIDTH (frame) - || height != GST_VIDEO_FRAME_HEIGHT (frame)) { + || width != GST_VIDEO_INFO_WIDTH (&pad->buffer_vinfo) + || height != GST_VIDEO_INFO_HEIGHT (&pad->buffer_vinfo)) { GstVideoInfo tmp_info; gst_video_info_set_format (&tmp_info, cpad->conversion_info.finfo->format, @@ -399,17 +392,16 @@ gst_compositor_pad_prepare_frame (GstVideoAggregatorPad * pad, tmp_info.interlace_mode = cpad->conversion_info.interlace_mode; GST_DEBUG_OBJECT (pad, "This pad will be converted from %d to %d", - GST_VIDEO_INFO_FORMAT (&frame->info), + GST_VIDEO_INFO_FORMAT (&pad->buffer_vinfo), GST_VIDEO_INFO_FORMAT (&tmp_info)); - cpad->convert = gst_video_converter_new (&frame->info, &tmp_info, NULL); + cpad->convert = + gst_video_converter_new (&pad->buffer_vinfo, &tmp_info, NULL); cpad->conversion_info = tmp_info; if (!cpad->convert) { GST_WARNING_OBJECT (pad, "No path found for conversion"); g_free (colorimetry); g_free (wanted_colorimetry); - gst_video_frame_unmap (frame); - g_slice_free (GstVideoFrame, frame); return FALSE; } } else { @@ -470,17 +462,24 @@ gst_compositor_pad_prepare_frame (GstVideoAggregatorPad * pad, if (frame_obscured) { converted_frame = NULL; - gst_video_frame_unmap (frame); - g_slice_free (GstVideoFrame, frame); goto done; } if (cpad->alpha == 0.0) { GST_DEBUG_OBJECT (vagg, "Pad has alpha 0.0, not converting frame"); converted_frame = NULL; - gst_video_frame_unmap (frame); - g_slice_free (GstVideoFrame, frame); - } else if (cpad->convert) { + goto done; + } + + frame = g_slice_new0 (GstVideoFrame); + + if (!gst_video_frame_map (frame, &pad->buffer_vinfo, pad->buffer, + GST_MAP_READ)) { + GST_WARNING_OBJECT (vagg, "Could not map input buffer"); + return FALSE; + } + + if (cpad->convert) { gint converted_size; converted_frame = g_slice_new0 (GstVideoFrame); From dbbfba76ea74640a7f580dc0a828e035af35aff0 Mon Sep 17 00:00:00 2001 From: Nirbheek Chauhan Date: Fri, 13 Mar 2015 06:10:52 +0530 Subject: [PATCH 164/381] tests: Add a check for the new compositor pad-is-obscured optimization We verify that all the buffers on an obscured sinkpad are skipped by overriding the map() function in the GstVideoMeta of the buffers to set a variable when called. We also test that the buffers do get mapped when they're not obscured. Blame^WCredit for the GstVideoMeta map() idea goes to Tim. https://bugzilla.gnome.org/show_bug.cgi?id=746147 --- tests/check/elements/compositor.c | 209 ++++++++++++++++++++++++++++++ 1 file changed, 209 insertions(+) diff --git a/tests/check/elements/compositor.c b/tests/check/elements/compositor.c index dab259bdf2..4ff890b66b 100644 --- a/tests/check/elements/compositor.c +++ b/tests/check/elements/compositor.c @@ -33,6 +33,7 @@ #include #include +#include #include #define VIDEO_CAPS_STRING \ @@ -1096,6 +1097,213 @@ GST_START_TEST (test_segment_base_handling) GST_END_TEST; +static gboolean buffer_mapped; +static gboolean (*default_map) (GstVideoMeta * meta, guint plane, + GstMapInfo * info, gpointer * data, gint * stride, GstMapFlags flags); + +static gboolean +test_obscured_new_videometa_map (GstVideoMeta * meta, guint plane, + GstMapInfo * info, gpointer * data, gint * stride, GstMapFlags flags) +{ + buffer_mapped = TRUE; + return default_map (meta, plane, info, data, stride, flags); +} + +static GstPadProbeReturn +test_obscured_pad_probe_cb (GstPad * srcpad, GstPadProbeInfo * info, + gpointer user_data) +{ + GstBuffer *obuf, *nbuf; + GstVideoMeta *meta; + + GST_DEBUG ("pad probe called"); + /* We need to deep-copy the buffer here because videotestsrc reuses buffers + * and hence the GstVideoMap associated with the buffers, and that causes a + * segfault inside videotestsrc when it tries to reuse the buffer */ + obuf = GST_PAD_PROBE_INFO_BUFFER (info); + nbuf = gst_buffer_new (); + gst_buffer_copy_into (nbuf, obuf, GST_BUFFER_COPY_ALL | GST_BUFFER_COPY_DEEP, + 0, -1); + meta = gst_buffer_get_video_meta (nbuf); + /* Override the default map() function to set also buffer_mapped */ + default_map = meta->map; + meta->map = test_obscured_new_videometa_map; + /* Replace the buffer that's going downstream */ + GST_PAD_PROBE_INFO_DATA (info) = nbuf; + gst_buffer_unref (obuf); + + return GST_PAD_PROBE_PASS; +} + +static void +_test_obscured (const gchar * caps_str, gint xpos0, gint ypos0, gint width0, + gint height0, gdouble alpha0, gint xpos1, gint ypos1, gint width1, + gint height1, gdouble alpha1) +{ + GstElement *pipeline, *sink, *mix, *src0, *cfilter0, *src1, *cfilter1; + GstPad *srcpad, *sinkpad; + GstSample *last_sample = NULL; + GstSample *sample; + GstCaps *caps; + + GST_INFO ("preparing test"); + + pipeline = gst_pipeline_new ("pipeline"); + src0 = gst_element_factory_make ("videotestsrc", "src0"); + g_object_set (src0, "num-buffers", 5, NULL); + cfilter0 = gst_element_factory_make ("capsfilter", "capsfilter0"); + caps = gst_caps_from_string (caps_str); + g_object_set (cfilter0, "caps", caps, NULL); + gst_caps_unref (caps); + + src1 = gst_element_factory_make ("videotestsrc", "src1"); + g_object_set (src1, "num-buffers", 5, NULL); + cfilter1 = gst_element_factory_make ("capsfilter", "capsfilter1"); + caps = gst_caps_from_string (caps_str); + g_object_set (cfilter1, "caps", caps, NULL); + gst_caps_unref (caps); + + mix = gst_element_factory_make ("compositor", "compositor"); + sink = gst_element_factory_make ("appsink", "sink"); + gst_bin_add_many (GST_BIN (pipeline), src0, cfilter0, src1, cfilter1, mix, + sink, NULL); + fail_unless (gst_element_link (src0, cfilter0)); + fail_unless (gst_element_link (src1, cfilter1)); + fail_unless (gst_element_link (mix, sink)); + + srcpad = gst_element_get_static_pad (cfilter0, "src"); + sinkpad = gst_element_get_request_pad (mix, "sink_0"); + g_object_set (sinkpad, "xpos", xpos0, "ypos", ypos0, "width", width0, + "height", height0, "alpha", alpha0, NULL); + fail_unless (gst_pad_link (srcpad, sinkpad) == GST_PAD_LINK_OK); + gst_pad_add_probe (srcpad, GST_PAD_PROBE_TYPE_BUFFER, + test_obscured_pad_probe_cb, NULL, NULL); + gst_object_unref (sinkpad); + gst_object_unref (srcpad); + + srcpad = gst_element_get_static_pad (cfilter1, "src"); + sinkpad = gst_element_get_request_pad (mix, "sink_1"); + g_object_set (sinkpad, "xpos", xpos1, "ypos", ypos1, "width", width1, + "height", height1, "alpha", alpha1, NULL); + fail_unless (gst_pad_link (srcpad, sinkpad) == GST_PAD_LINK_OK); + gst_object_unref (sinkpad); + gst_object_unref (srcpad); + + GST_INFO ("sample prepared"); + gst_element_set_state (pipeline, GST_STATE_PLAYING); + + do { + GST_DEBUG ("sample pulling"); + g_signal_emit_by_name (sink, "pull-sample", &sample); + if (sample == NULL) + break; + if (last_sample) + gst_sample_unref (last_sample); + last_sample = sample; + GST_DEBUG ("sample pulled"); + } while (TRUE); + gst_sample_unref (last_sample); + + gst_element_set_state (pipeline, GST_STATE_NULL); + gst_object_unref (pipeline); +} + +GST_START_TEST (test_obscured_skipped) +{ + gint xpos0, xpos1; + gint ypos0, ypos1; + gint width0, width1; + gint height0, height1; + gdouble alpha0, alpha1; + const gchar *caps_str; + + caps_str = "video/x-raw"; + buffer_mapped = FALSE; + /* Set else to compositor defaults */ + alpha0 = alpha1 = 1.0; + xpos0 = xpos1 = ypos0 = ypos1 = 0; + width0 = width1 = height0 = height1 = 0; + + GST_INFO ("testing defaults"); + /* With everything at defaults, sink_1 will obscure sink_0, so buffers from + * sink_0 will never get mapped by compositor. To verify, run with + * GST_DEBUG=compositor:6 and look for "Obscured by" messages */ + _test_obscured (caps_str, xpos0, ypos0, width0, height0, alpha0, xpos1, ypos1, + width1, height1, alpha1); + fail_unless (buffer_mapped == FALSE); + buffer_mapped = FALSE; + + caps_str = "video/x-raw,format=ARGB"; + GST_INFO ("testing video with alpha channel"); + _test_obscured (caps_str, xpos0, ypos0, width0, height0, alpha0, xpos1, ypos1, + width1, height1, alpha1); + fail_unless (buffer_mapped == TRUE); + caps_str = "video/x-raw"; + buffer_mapped = FALSE; + + alpha1 = 0.0; + GST_INFO ("testing alpha1 = %.2g", alpha1); + _test_obscured (caps_str, xpos0, ypos0, width0, height0, alpha0, xpos1, ypos1, + width1, height1, alpha1); + fail_unless (buffer_mapped == TRUE); + alpha1 = 1.0; + buffer_mapped = FALSE; + + /* Test 0.1, ..., 0.9 */ + for (alpha1 = 1; alpha1 < 10; alpha1 += 1) { + GST_INFO ("testing alpha1 = %.2g", alpha1 / 10); + _test_obscured (caps_str, xpos0, ypos0, width0, height0, alpha0, xpos1, + ypos1, width1, height1, alpha1 / 10); + fail_unless (buffer_mapped == TRUE); + } + alpha1 = 1.0; + buffer_mapped = FALSE; + + width1 = height1 = 10; + GST_INFO ("testing smaller sink_1"); + _test_obscured (caps_str, xpos0, ypos0, width0, height0, alpha0, xpos1, ypos1, + width1, height1, alpha1); + fail_unless (buffer_mapped == TRUE); + width1 = height1 = 0; + buffer_mapped = FALSE; + + width0 = height0 = width1 = height1 = 10; + GST_INFO ("testing smaller sink_1 and sink0 (same sizes)"); + _test_obscured (caps_str, xpos0, ypos0, width0, height0, alpha0, xpos1, ypos1, + width1, height1, alpha1); + fail_unless (buffer_mapped == FALSE); + width0 = height0 = width1 = height1 = 0; + buffer_mapped = FALSE; + + width0 = height0 = 20; + width1 = height1 = 10; + GST_INFO ("testing smaller sink_1 and sink0 (sink_0 > sink_1)"); + _test_obscured (caps_str, xpos0, ypos0, width0, height0, alpha0, xpos1, ypos1, + width1, height1, alpha1); + fail_unless (buffer_mapped == TRUE); + width0 = height0 = width1 = height1 = 0; + buffer_mapped = FALSE; + + width0 = height0 = 10; + width1 = height1 = 20; + GST_INFO ("testing smaller sink_1 and sink0 (sink_0 < sink_1)"); + _test_obscured (caps_str, xpos0, ypos0, width0, height0, alpha0, xpos1, ypos1, + width1, height1, alpha1); + fail_unless (buffer_mapped == FALSE); + width0 = height0 = width1 = height1 = 0; + buffer_mapped = FALSE; + + xpos0 = ypos0 = 10000; + GST_INFO ("testing sink_0 outside the frame"); + _test_obscured (caps_str, xpos0, ypos0, width0, height0, alpha0, xpos1, ypos1, + width1, height1, alpha1); + fail_unless (buffer_mapped == FALSE); + xpos0 = ypos0 = 0; + buffer_mapped = FALSE; +} + +GST_END_TEST; + static Suite * compositor_suite (void) { @@ -1115,6 +1323,7 @@ compositor_suite (void) tcase_add_test (tc_chain, test_loop); tcase_add_test (tc_chain, test_flush_start_flush_stop); tcase_add_test (tc_chain, test_segment_base_handling); + tcase_add_test (tc_chain, test_obscured_skipped); /* Use a longer timeout */ #ifdef HAVE_VALGRIND From 824bb6acf82842f26be88ed0d166c12e4deea264 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Wed, 6 May 2015 15:43:32 +0200 Subject: [PATCH 165/381] Revert "Revert "glvideomixer: implement with glmixerbin"" This reverts commit b4bd11f2f3a60224d188b27ab55b278077cb1217. --- ext/gl/gstglvideomixer.c | 313 ++++++++++++++++++++++++++++++++------- ext/gl/gstglvideomixer.h | 1 + 2 files changed, 263 insertions(+), 51 deletions(-) diff --git a/ext/gl/gstglvideomixer.c b/ext/gl/gstglvideomixer.c index b519995cdd..cbba72a5d7 100644 --- a/ext/gl/gstglvideomixer.c +++ b/ext/gl/gstglvideomixer.c @@ -44,16 +44,273 @@ #endif #include "gstglvideomixer.h" +#include "gstglmixerbin.h" #define GST_CAT_DEFAULT gst_gl_video_mixer_debug GST_DEBUG_CATEGORY_STATIC (GST_CAT_DEFAULT); +#define GST_GL_TYPE_VIDEO_MIXER_BACKGROUND (gst_gl_video_mixer_background_get_type()) +static GType +gst_gl_video_mixer_background_get_type (void) +{ + static GType mixer_background_type = 0; + + static const GEnumValue mixer_background[] = { + {GST_GL_VIDEO_MIXER_BACKGROUND_CHECKER, "Checker pattern", "checker"}, + {GST_GL_VIDEO_MIXER_BACKGROUND_BLACK, "Black", "black"}, + {GST_GL_VIDEO_MIXER_BACKGROUND_WHITE, "White", "white"}, + {GST_GL_VIDEO_MIXER_BACKGROUND_TRANSPARENT, + "Transparent Background to enable further compositing", "transparent"}, + {0, NULL, NULL}, + }; + + if (!mixer_background_type) { + mixer_background_type = + g_enum_register_static ("GstGLVideoMixerBackground", mixer_background); + } + return mixer_background_type; +} + +#define DEFAULT_PAD_XPOS 0 +#define DEFAULT_PAD_YPOS 0 +#define DEFAULT_PAD_WIDTH 0 +#define DEFAULT_PAD_HEIGHT 0 +#define DEFAULT_PAD_ALPHA 1.0 +#define DEFAULT_PAD_ZORDER 0 + +enum +{ + PROP_INPUT_0, + PROP_INPUT_XPOS, + PROP_INPUT_YPOS, + PROP_INPUT_WIDTH, + PROP_INPUT_HEIGHT, + PROP_INPUT_ALPHA, + PROP_INPUT_ZORDER, +}; + +static void gst_gl_video_mixer_input_get_property (GObject * object, + guint prop_id, GValue * value, GParamSpec * pspec); +static void gst_gl_video_mixer_input_set_property (GObject * object, + guint prop_id, const GValue * value, GParamSpec * pspec); + +static GstFlowReturn gst_gl_video_mixer_input_chain (GstPad * pad, + GstObject * parent, GstBuffer * buffer); +static GstFlowReturn gst_gl_video_mixer_input_event (GstPad * pad, + GstObject * parent, GstEvent * event); + +typedef struct _GstGLVideoMixerInput GstGLVideoMixerInput; +typedef GstGhostPadClass GstGLVideoMixerInputClass; + +struct _GstGLVideoMixerInput +{ + GstGhostPad parent; + + GstSegment segment; + + GstPad *mixer_pad; +}; + +GType gst_gl_video_mixer_input_get_type (void); +G_DEFINE_TYPE (GstGLVideoMixerInput, gst_gl_video_mixer_input, + GST_TYPE_GHOST_PAD); + +static void +gst_gl_video_mixer_input_init (GstGLVideoMixerInput * self) +{ + GstPad *pad = GST_PAD (self); + + gst_pad_set_event_function (pad, gst_gl_video_mixer_input_event); +} + +static void +gst_gl_video_mixer_input_class_init (GstGLVideoMixerInputClass * klass) +{ + GObjectClass *gobject_class = G_OBJECT_CLASS (klass); + + gobject_class->set_property = gst_gl_video_mixer_input_set_property; + gobject_class->get_property = gst_gl_video_mixer_input_get_property; + + g_object_class_install_property (gobject_class, PROP_INPUT_ZORDER, + g_param_spec_uint ("zorder", "Z-Order", "Z Order of the picture", + 0, 10000, DEFAULT_PAD_ZORDER, + G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); + g_object_class_install_property (gobject_class, PROP_INPUT_XPOS, + g_param_spec_int ("xpos", "X Position", "X Position of the picture", + G_MININT, G_MAXINT, DEFAULT_PAD_XPOS, + G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); + g_object_class_install_property (gobject_class, PROP_INPUT_YPOS, + g_param_spec_int ("ypos", "Y Position", "Y Position of the picture", + G_MININT, G_MAXINT, DEFAULT_PAD_YPOS, + G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); + g_object_class_install_property (gobject_class, PROP_INPUT_WIDTH, + g_param_spec_int ("width", "Width", "Width of the picture", + G_MININT, G_MAXINT, DEFAULT_PAD_WIDTH, + G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); + g_object_class_install_property (gobject_class, PROP_INPUT_HEIGHT, + g_param_spec_int ("height", "Height", "Height of the picture", + G_MININT, G_MAXINT, DEFAULT_PAD_HEIGHT, + G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); + g_object_class_install_property (gobject_class, PROP_INPUT_ALPHA, + g_param_spec_double ("alpha", "Alpha", "Alpha of the picture", 0.0, 1.0, + DEFAULT_PAD_ALPHA, + G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); +} + +static void +gst_gl_video_mixer_input_get_property (GObject * object, guint prop_id, + GValue * value, GParamSpec * pspec) +{ + GstGLVideoMixerInput *self = (GstGLVideoMixerInput *) object; + + if (self->mixer_pad) + g_object_get_property (G_OBJECT (self->mixer_pad), pspec->name, value); +} + +static void +gst_gl_video_mixer_input_set_property (GObject * object, guint prop_id, + const GValue * value, GParamSpec * pspec) +{ + GstGLVideoMixerInput *self = (GstGLVideoMixerInput *) object; + + if (self->mixer_pad) + g_object_set_property (G_OBJECT (self->mixer_pad), pspec->name, value); +} + +static GstFlowReturn +gst_gl_video_mixer_input_chain (GstPad * pad, GstObject * parent, + GstBuffer * buffer) +{ + GstGLVideoMixerInput *self = (GstGLVideoMixerInput *) pad; + GstClockTime timestamp, stream_time; +// gdouble alpha; + + timestamp = GST_BUFFER_TIMESTAMP (buffer); + + stream_time = + gst_segment_to_stream_time (&self->segment, GST_FORMAT_TIME, timestamp); + + gst_object_sync_values (GST_OBJECT (self), stream_time); +#if 0 + /* FIXME: implement no-upload on alpha = 0 */ + g_object_get (self, "alpha", &alpha, NULL); + + if (alpha <= 0.0) { + GST_DEBUG_OBJECT (self, "dropping buffer %" GST_PTR_FORMAT + " due to alpha value %f", buffer, alpha); + gst_buffer_unref (buffer); + return GST_FLOW_OK; + } +#endif + return gst_proxy_pad_chain_default (pad, parent, buffer); +} + +static GstFlowReturn +gst_gl_video_mixer_input_event (GstPad * pad, GstObject * parent, + GstEvent * event) +{ + GstGLVideoMixerInput *self = (GstGLVideoMixerInput *) pad; + + switch (GST_EVENT_TYPE (event)) { + case GST_EVENT_SEGMENT: + gst_event_copy_segment (event, &self->segment); + break; + default: + break; + } + + return gst_pad_event_default (pad, parent, event); +} + +static GstGhostPad * +_create_video_mixer_input (GstGLMixerBin * self, GstPad * mixer_pad) +{ + GstGLVideoMixerInput *input = + g_object_new (gst_gl_video_mixer_input_get_type (), "name", + GST_OBJECT_NAME (mixer_pad), "direction", GST_PAD_DIRECTION (mixer_pad), + NULL); + + if (!gst_ghost_pad_construct (GST_GHOST_PAD (input))) { + gst_object_unref (input); + return NULL; + } + + gst_pad_set_chain_function (GST_PAD (input), gst_gl_video_mixer_input_chain); + + input->mixer_pad = mixer_pad; + + return GST_GHOST_PAD (input); +} + +enum +{ + PROP_BIN_0, + PROP_BIN_BACKGROUND, +}; +#define DEFAULT_BACKGROUND GST_GL_VIDEO_MIXER_BACKGROUND_CHECKER + +static void gst_gl_video_mixer_bin_get_property (GObject * object, + guint prop_id, GValue * value, GParamSpec * pspec); +static void gst_gl_video_mixer_bin_set_property (GObject * object, + guint prop_id, const GValue * value, GParamSpec * pspec); + +typedef GstGLMixerBin GstGLVideoMixerBin; +typedef GstGLMixerBinClass GstGLVideoMixerBinClass; + +G_DEFINE_TYPE (GstGLVideoMixerBin, gst_gl_video_mixer_bin, + GST_TYPE_GL_MIXER_BIN); + +static void +gst_gl_video_mixer_bin_init (GstGLVideoMixerBin * self) +{ + GstGLMixerBin *mix_bin = GST_GL_MIXER_BIN (self); + + gst_gl_mixer_bin_finish_init_with_element (mix_bin, + g_object_new (GST_TYPE_GL_VIDEO_MIXER, NULL)); +} + +static void +gst_gl_video_mixer_bin_class_init (GstGLVideoMixerBinClass * klass) +{ + GstGLMixerBinClass *mixer_class = GST_GL_MIXER_BIN_CLASS (klass); + GObjectClass *gobject_class = G_OBJECT_CLASS (klass); + + mixer_class->create_input_pad = _create_video_mixer_input; + + gobject_class->set_property = gst_gl_video_mixer_bin_set_property; + gobject_class->get_property = gst_gl_video_mixer_bin_get_property; + + g_object_class_install_property (gobject_class, PROP_BIN_BACKGROUND, + g_param_spec_enum ("background", "Background", "Background type", + GST_GL_TYPE_VIDEO_MIXER_BACKGROUND, + DEFAULT_BACKGROUND, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); +} + +static void +gst_gl_video_mixer_bin_get_property (GObject * object, guint prop_id, + GValue * value, GParamSpec * pspec) +{ + GstGLMixerBin *self = GST_GL_MIXER_BIN (object); + + if (self->mixer) + g_object_get_property (G_OBJECT (self->mixer), pspec->name, value); +} + +static void +gst_gl_video_mixer_bin_set_property (GObject * object, guint prop_id, + const GValue * value, GParamSpec * pspec) +{ + GstGLMixerBin *self = GST_GL_MIXER_BIN (object); + + if (self->mixer) + g_object_set_property (G_OBJECT (self->mixer), pspec->name, value); +} + enum { PROP_0, PROP_BACKGROUND, }; -#define DEFAULT_BACKGROUND GST_GL_VIDEO_MIXER_BACKGROUND_CHECKER #define DEBUG_INIT \ GST_DEBUG_CATEGORY_INIT (gst_gl_video_mixer_debug, "glvideomixer", 0, "glvideomixer element"); @@ -178,14 +435,7 @@ static void gst_gl_video_mixer_pad_set_property (GObject * object, guint prop_id, const GValue * value, GParamSpec * pspec); static void gst_gl_video_mixer_pad_get_property (GObject * object, guint prop_id, GValue * value, GParamSpec * pspec); -static GstBuffer *gst_gl_video_mixer_pad_upload_buffer (GstGLMixer * mix, - GstGLMixerFrameData * frame, GstBuffer * buffer); -#define DEFAULT_PAD_XPOS 0 -#define DEFAULT_PAD_YPOS 0 -#define DEFAULT_PAD_WIDTH 0 -#define DEFAULT_PAD_HEIGHT 0 -#define DEFAULT_PAD_ALPHA 1.0 enum { PROP_PAD_0, @@ -206,7 +456,6 @@ static void gst_gl_video_mixer_pad_class_init (GstGLVideoMixerPadClass * klass) { GObjectClass *gobject_class = (GObjectClass *) klass; - GstGLMixerPadClass *mix_pad_class = (GstGLMixerPadClass *) klass; gobject_class->set_property = gst_gl_video_mixer_pad_set_property; gobject_class->get_property = gst_gl_video_mixer_pad_get_property; @@ -231,8 +480,6 @@ gst_gl_video_mixer_pad_class_init (GstGLVideoMixerPadClass * klass) g_param_spec_double ("alpha", "Alpha", "Alpha of the picture", 0.0, 1.0, DEFAULT_PAD_ALPHA, G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); - - mix_pad_class->upload_buffer = gst_gl_video_mixer_pad_upload_buffer; } static void @@ -298,44 +545,6 @@ gst_gl_video_mixer_pad_set_property (GObject * object, guint prop_id, gst_object_unref (mix); } -static GstBuffer * -gst_gl_video_mixer_pad_upload_buffer (GstGLMixer * mix, - GstGLMixerFrameData * frame, GstBuffer * buffer) -{ - GstGLVideoMixerPad *pad = GST_GL_VIDEO_MIXER_PAD (frame->pad); - - if (pad->alpha > 0.0) { - return - GST_GL_MIXER_PAD_CLASS - (gst_gl_video_mixer_pad_parent_class)->upload_buffer (mix, frame, - buffer); - } - - return NULL; -} - -#define GST_GL_TYPE_VIDEO_MIXER_BACKGROUND (gst_gl_video_mixer_background_get_type()) -static GType -gst_gl_video_mixer_background_get_type (void) -{ - static GType mixer_background_type = 0; - - static const GEnumValue mixer_background[] = { - {GST_GL_VIDEO_MIXER_BACKGROUND_CHECKER, "Checker pattern", "checker"}, - {GST_GL_VIDEO_MIXER_BACKGROUND_BLACK, "Black", "black"}, - {GST_GL_VIDEO_MIXER_BACKGROUND_WHITE, "White", "white"}, - {GST_GL_VIDEO_MIXER_BACKGROUND_TRANSPARENT, - "Transparent Background to enable further compositing", "transparent"}, - {0, NULL, NULL}, - }; - - if (!mixer_background_type) { - mixer_background_type = - g_enum_register_static ("GstGLVideoMixerBackground", mixer_background); - } - return mixer_background_type; -} - static void gst_gl_video_mixer_class_init (GstGLVideoMixerClass * klass) { @@ -352,7 +561,7 @@ gst_gl_video_mixer_class_init (GstGLVideoMixerClass * klass) gst_element_class_set_metadata (element_class, "OpenGL video_mixer", "Filter/Effect/Video/Compositor", "OpenGL video_mixer", - "Julien Isorce "); + "Matthew Waters "); g_object_class_install_property (gobject_class, PROP_BACKGROUND, g_param_spec_enum ("background", "Background", "Background type", @@ -454,7 +663,9 @@ _update_caps (GstVideoAggregator * vagg, GstCaps * caps) } GST_OBJECT_UNLOCK (vagg); - ret = gst_gl_mixer_update_caps (GST_GL_MIXER (vagg), caps); + ret = + GST_VIDEO_AGGREGATOR_CLASS (gst_gl_video_mixer_parent_class)->update_caps + (vagg, caps); for (i = 0; i < gst_caps_get_size (ret); i++) { GstStructure *s = gst_caps_get_structure (ret, i); diff --git a/ext/gl/gstglvideomixer.h b/ext/gl/gstglvideomixer.h index b351888fce..967358baba 100644 --- a/ext/gl/gstglvideomixer.h +++ b/ext/gl/gstglvideomixer.h @@ -73,6 +73,7 @@ struct _GstGLVideoMixerClass }; GType gst_gl_video_mixer_get_type (void); +GType gst_gl_video_mixer_bin_get_type (void); G_END_DECLS From ee275a167d25e3ecafc718fffd6159cbccb152a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Wed, 6 May 2015 15:46:49 +0200 Subject: [PATCH 166/381] Revert "gl: readd glupload/download onto element pads" This reverts commit 87d8270f302b03f63ce04f986d824892a2c131fd. --- ext/gl/gstglmixer.c | 178 +++++++++---------------------------------- ext/gl/gstglmixer.h | 8 -- ext/gl/gstglmosaic.c | 3 - 3 files changed, 35 insertions(+), 154 deletions(-) diff --git a/ext/gl/gstglmixer.c b/ext/gl/gstglmixer.c index 907a160dc4..8e35af10b0 100644 --- a/ext/gl/gstglmixer.c +++ b/ext/gl/gstglmixer.c @@ -42,9 +42,6 @@ static void gst_gl_mixer_pad_get_property (GObject * object, guint prop_id, GValue * value, GParamSpec * pspec); static void gst_gl_mixer_pad_set_property (GObject * object, guint prop_id, const GValue * value, GParamSpec * pspec); -static void gst_gl_mixer_pad_finalize (GObject * object); -static GstBuffer *_default_pad_upload_buffer (GstGLMixer * mix, - GstGLMixerFrameData * frame, GstBuffer * buffer); enum { @@ -74,13 +71,10 @@ gst_gl_mixer_pad_class_init (GstGLMixerPadClass * klass) gobject_class->set_property = gst_gl_mixer_pad_set_property; gobject_class->get_property = gst_gl_mixer_pad_get_property; - gobject_class->finalize = gst_gl_mixer_pad_finalize; vaggpad_class->set_info = NULL; vaggpad_class->prepare_frame = NULL; vaggpad_class->clean_frame = NULL; - - klass->upload_buffer = _default_pad_upload_buffer; } static void @@ -204,22 +198,12 @@ gst_gl_mixer_pad_sink_acceptcaps (GstPad * pad, GstGLMixer * mix, return ret; } -GstCaps * -gst_gl_mixer_update_caps (GstGLMixer * mix, GstCaps * caps) +/* copies the given caps */ +static GstCaps * +_update_caps (GstVideoAggregator * vagg, GstCaps * caps) { - GstGLContext *context = GST_GL_BASE_MIXER (mix)->context; - GstCaps *result, *tmp; - - result = - gst_caps_from_string (GST_VIDEO_CAPS_MAKE_WITH_FEATURES - (GST_CAPS_FEATURE_MEMORY_GL_MEMORY, "RGBA")); - - tmp = gst_gl_upload_transform_caps (context, GST_PAD_SRC, result, NULL); - gst_caps_unref (result); - result = tmp; - GST_DEBUG_OBJECT (mix, "transfer returned caps %" GST_PTR_FORMAT, result); - - return result; + return gst_gl_caps_replace_all_caps_features (caps, + GST_CAPS_FEATURE_MEMORY_GL_MEMORY); } static GstCaps * @@ -300,17 +284,6 @@ gst_gl_mixer_pad_init (GstGLMixerPad * mixerpad) { } -static void -gst_gl_mixer_pad_finalize (GObject * object) -{ - GstGLMixerPad *pad = GST_GL_MIXER_PAD (object); - - gst_buffer_replace (&pad->uploaded_buffer, NULL); - gst_object_replace ((GstObject **) & pad->upload, NULL); - - G_OBJECT_CLASS (gst_gl_mixer_pad_parent_class)->finalize (object); -} - /* GLMixer signals and args */ enum { @@ -328,7 +301,15 @@ static GstStaticPadTemplate src_factory = GST_STATIC_PAD_TEMPLATE ("src", GST_PAD_ALWAYS, GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES (GST_CAPS_FEATURE_MEMORY_GL_MEMORY, - "RGBA") ";" GST_VIDEO_CAPS_MAKE ("RGBA")) + "RGBA")) + ); + +static GstStaticPadTemplate sink_factory = GST_STATIC_PAD_TEMPLATE ("sink_%u", + GST_PAD_SINK, + GST_PAD_REQUEST, + GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES + (GST_CAPS_FEATURE_MEMORY_GL_MEMORY, + "RGBA")) ); static gboolean gst_gl_mixer_src_query (GstAggregator * agg, GstQuery * query); @@ -359,8 +340,7 @@ gst_gl_mixer_class_init (GstGLMixerClass * klass) GstVideoAggregatorClass *videoaggregator_class = (GstVideoAggregatorClass *) klass; GstAggregatorClass *agg_class = (GstAggregatorClass *) klass; - GstGLBaseMixerClass *mix_class = GST_GL_BASE_MIXER_CLASS (klass); - GstCaps *upload_caps, *elem_caps, *templ; + GstGLBaseMixerClass *mix_class = GST_GL_BASE_MIXER_CLASS (klass);; GST_DEBUG_CATEGORY_INIT (GST_CAT_DEFAULT, "glmixer", 0, "OpenGL mixer"); @@ -373,15 +353,8 @@ gst_gl_mixer_class_init (GstGLMixerClass * klass) gst_element_class_add_pad_template (element_class, gst_static_pad_template_get (&src_factory)); - - elem_caps = gst_caps_from_string ("video/x-raw(ANY),format=RGBA"); - upload_caps = gst_gl_upload_get_input_template_caps (); - templ = gst_caps_intersect (elem_caps, upload_caps); gst_element_class_add_pad_template (element_class, - gst_pad_template_new ("sink_%u", GST_PAD_SINK, GST_PAD_REQUEST, templ)); - gst_caps_unref (upload_caps); - gst_caps_unref (elem_caps); - gst_caps_unref (templ); + gst_static_pad_template_get (&sink_factory)); agg_class->sinkpads_type = GST_TYPE_GL_MIXER_PAD; agg_class->sink_query = gst_gl_mixer_sink_query; @@ -392,6 +365,7 @@ gst_gl_mixer_class_init (GstGLMixerClass * klass) videoaggregator_class->aggregate_frames = gst_gl_mixer_aggregate_frames; videoaggregator_class->get_output_buffer = gst_gl_mixer_get_output_buffer; videoaggregator_class->negotiated_caps = _negotiated_caps; + videoaggregator_class->update_caps = _update_caps; videoaggregator_class->find_best_format = NULL; mix_class->propose_allocation = gst_gl_mixer_propose_allocation; @@ -571,11 +545,8 @@ gst_gl_mixer_decide_allocation (GstGLBaseMixer * base_mix, GstQuery * query) update_pool = FALSE; } - if (!pool || !GST_IS_GL_BUFFER_POOL (pool)) { - if (pool) - gst_object_unref (pool); + if (!pool) pool = gst_gl_buffer_pool_new (context); - } config = gst_buffer_pool_get_config (pool); gst_buffer_pool_config_set_params (config, caps, size, min, max); @@ -599,63 +570,6 @@ context_error: } } -static GstBuffer * -_default_pad_upload_buffer (GstGLMixer * mix, GstGLMixerFrameData * frame, - GstBuffer * buffer) -{ - GstVideoAggregatorPad *vaggpad = GST_VIDEO_AGGREGATOR_PAD (frame->pad); - GstGLMixerPad *pad = frame->pad; - GstVideoInfo gl_info; - GstGLSyncMeta *sync_meta; - - gst_video_info_set_format (&gl_info, - GST_VIDEO_FORMAT_RGBA, - GST_VIDEO_INFO_WIDTH (&vaggpad->info), - GST_VIDEO_INFO_HEIGHT (&vaggpad->info)); - - if (!pad->upload) { - GstCaps *in_caps, *upload_caps; - - in_caps = gst_pad_get_current_caps (GST_PAD (pad)); - upload_caps = gst_video_info_to_caps (&gl_info); - gst_caps_set_features (upload_caps, 0, - gst_caps_features_from_string (GST_CAPS_FEATURE_MEMORY_GL_MEMORY)); - - pad->upload = gst_gl_upload_new (GST_GL_BASE_MIXER (mix)->context); - - if (!gst_gl_upload_set_caps (pad->upload, in_caps, upload_caps)) { - GST_WARNING_OBJECT (pad, "Failed to configure uploader"); - gst_caps_unref (in_caps); - gst_caps_unref (upload_caps); - gst_object_unref (pad->upload); - pad->upload = NULL; - return NULL; - } - - gst_caps_unref (in_caps); - gst_caps_unref (upload_caps); - } - - sync_meta = gst_buffer_get_gl_sync_meta (vaggpad->buffer); - if (sync_meta) - gst_gl_sync_meta_wait (sync_meta, GST_GL_BASE_MIXER (mix)->context); - - if (GST_GL_UPLOAD_DONE != gst_gl_upload_perform_with_buffer (pad->upload, - vaggpad->buffer, &pad->uploaded_buffer) || !pad->uploaded_buffer) { - GST_WARNING_OBJECT (pad, "Failed to upload input buffer %p", - vaggpad->buffer); - return NULL; - } - - if (gst_video_frame_map (&frame->v_frame, &gl_info, pad->uploaded_buffer, - GST_MAP_READ | GST_MAP_GL)) { - frame->texture = *(guint *) frame->v_frame.data[0]; - frame->mapped = TRUE; - } - - return pad->uploaded_buffer; -} - gboolean gst_gl_mixer_process_textures (GstGLMixer * mix, GstBuffer * outbuf) { @@ -663,7 +577,7 @@ gst_gl_mixer_process_textures (GstGLMixer * mix, GstBuffer * outbuf) GList *walk; guint out_tex; gboolean res = TRUE; - gint array_index = 0; + guint array_index = 0; GstVideoFrame out_frame; GstElement *element = GST_ELEMENT (mix); GstVideoAggregator *vagg = GST_VIDEO_AGGREGATOR (mix); @@ -688,28 +602,34 @@ gst_gl_mixer_process_textures (GstGLMixer * mix, GstBuffer * outbuf) mix->frames->pdata[i] = g_slice_new0 (GstGLMixerFrameData); while (walk) { GstGLMixerPad *pad = GST_GL_MIXER_PAD (walk->data); - GstGLMixerPadClass *pad_class = GST_GL_MIXER_PAD_GET_CLASS (pad); GstVideoAggregatorPad *vaggpad = walk->data; GstGLMixerFrameData *frame; frame = g_ptr_array_index (mix->frames, array_index); frame->pad = pad; frame->texture = 0; - frame->mapped = FALSE; walk = g_list_next (walk); if (vaggpad->buffer != NULL) { - g_assert (pad_class->upload_buffer); + GstVideoInfo gl_info; + GstVideoFrame gl_frame; + GstGLSyncMeta *sync_meta; - if (pad->uploaded_buffer) - gst_buffer_unref (pad->uploaded_buffer); - pad->uploaded_buffer = - pad_class->upload_buffer (mix, frame, vaggpad->buffer); + gst_video_info_set_format (&gl_info, + GST_VIDEO_FORMAT_RGBA, + GST_VIDEO_INFO_WIDTH (&vaggpad->info), + GST_VIDEO_INFO_HEIGHT (&vaggpad->info)); - GST_DEBUG_OBJECT (pad, - "uploaded buffer %" GST_PTR_FORMAT " from buffer %" GST_PTR_FORMAT, - pad->uploaded_buffer, vaggpad->buffer); + sync_meta = gst_buffer_get_gl_sync_meta (vaggpad->buffer); + if (sync_meta) + gst_gl_sync_meta_wait (sync_meta, GST_GL_BASE_MIXER (mix)->context); + + if (gst_video_frame_map (&gl_frame, &gl_info, vaggpad->buffer, + GST_MAP_READ | GST_MAP_GL)) { + frame->texture = *(guint *) gl_frame.data[0]; + gst_video_frame_unmap (&gl_frame); + } } ++array_index; @@ -732,19 +652,6 @@ gst_gl_mixer_process_textures (GstGLMixer * mix, GstBuffer * outbuf) g_mutex_unlock (&priv->gl_resource_lock); out: - for (--array_index; array_index >= 0; array_index--) { - GstGLMixerFrameData *frame; - - frame = g_ptr_array_index (mix->frames, array_index); - - if (frame->mapped) - gst_video_frame_unmap (&frame->v_frame); - frame->mapped = FALSE; - frame->texture = 0; - - gst_buffer_replace (&frame->pad->uploaded_buffer, NULL); - } - GST_OBJECT_UNLOCK (mix); gst_video_frame_unmap (&out_frame); @@ -854,18 +761,6 @@ gst_gl_mixer_start (GstAggregator * agg) return GST_AGGREGATOR_CLASS (parent_class)->start (agg); } -static gboolean -_clean_upload (GstAggregator * agg, GstAggregatorPad * aggpad, - gpointer user_data) -{ - GstGLMixerPad *pad = GST_GL_MIXER_PAD (aggpad); - - gst_buffer_replace (&pad->uploaded_buffer, NULL); - gst_object_replace ((GstObject **) & pad->upload, NULL); - - return TRUE; -} - static gboolean gst_gl_mixer_stop (GstAggregator * agg) { @@ -873,9 +768,6 @@ gst_gl_mixer_stop (GstAggregator * agg) GstGLMixerClass *mixer_class = GST_GL_MIXER_GET_CLASS (mix); GstGLContext *context = GST_GL_BASE_MIXER (mix)->context; - gst_aggregator_iterate_sinkpads (agg, - (GstAggregatorPadForeachFunc) _clean_upload, NULL); - GST_OBJECT_LOCK (agg); g_ptr_array_free (mix->frames, TRUE); mix->frames = NULL; diff --git a/ext/gl/gstglmixer.h b/ext/gl/gstglmixer.h index 38b51d369d..01eed34f20 100644 --- a/ext/gl/gstglmixer.h +++ b/ext/gl/gstglmixer.h @@ -52,16 +52,11 @@ typedef struct _GstGLMixerPadClass GstGLMixerPadClass; struct _GstGLMixerPad { GstGLBaseMixerPad parent; - - GstGLUpload *upload; - GstBuffer *uploaded_buffer; }; struct _GstGLMixerPadClass { GstGLBaseMixerPadClass parent_class; - - GstBuffer * (*upload_buffer) (GstGLMixer * mix, GstGLMixerFrameData * frame, GstBuffer * buffer); }; GType gst_gl_mixer_pad_get_type (void); @@ -114,15 +109,12 @@ struct _GstGLMixerClass struct _GstGLMixerFrameData { GstGLMixerPad *pad; - gboolean mapped; - GstVideoFrame v_frame; guint texture; }; GType gst_gl_mixer_get_type(void); gboolean gst_gl_mixer_process_textures (GstGLMixer * mix, GstBuffer * outbuf); -GstCaps * gst_gl_mixer_update_caps (GstGLMixer * mix, GstCaps * caps); G_END_DECLS #endif /* __GST_GL_MIXER_H__ */ diff --git a/ext/gl/gstglmosaic.c b/ext/gl/gstglmosaic.c index 0dc13b7c67..726674b4df 100644 --- a/ext/gl/gstglmosaic.c +++ b/ext/gl/gstglmosaic.c @@ -190,9 +190,6 @@ gst_gl_mosaic_init_shader (GstGLMixer * mixer, GstCaps * outcaps) { GstGLMosaic *mosaic = GST_GL_MOSAIC (mixer); - if (mosaic->shader) - gst_object_unref (mosaic->shader); - //blocking call, wait the opengl thread has compiled the shader return gst_gl_context_gen_shader (GST_GL_BASE_MIXER (mixer)->context, mosaic_v_src, mosaic_f_src, &mosaic->shader); From fa6dbaeb6c71d51844fb25a19cfbd41d18703ad8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Mon, 11 May 2015 15:54:52 +0300 Subject: [PATCH 167/381] glmixer: Implement GstVideoAggregator::find_best_format() Without this, we will fixate weird pixel-aspect-ratios like 1/2147483647. But in the end, all the negotiation code in videoaggregator needs a big cleanup and videoaggregator needs to get rid of the software-mixer specific things everywhere. --- ext/gl/gstglmixer.c | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/ext/gl/gstglmixer.c b/ext/gl/gstglmixer.c index 8e35af10b0..119ab5a359 100644 --- a/ext/gl/gstglmixer.c +++ b/ext/gl/gstglmixer.c @@ -114,6 +114,26 @@ _negotiated_caps (GstVideoAggregator * vagg, GstCaps * caps) return ret; } +static void +_find_best_format (GstVideoAggregator * vagg, GstCaps * downstream_caps, + GstVideoInfo * best_info, gboolean * at_least_one_alpha) +{ + GstVideoInfo tmp_info; + + GST_VIDEO_AGGREGATOR_CLASS (parent_class)->find_best_format (vagg, + downstream_caps, best_info, at_least_one_alpha); + + gst_video_info_set_format (&tmp_info, GST_VIDEO_FORMAT_RGBA, + best_info->width, best_info->height); + tmp_info.par_n = best_info->par_n; + tmp_info.par_d = best_info->par_d; + tmp_info.fps_n = best_info->fps_n; + tmp_info.fps_d = best_info->fps_d; + tmp_info.flags = best_info->flags; + tmp_info.interlace_mode = best_info->interlace_mode; + *best_info = tmp_info; +} + static gboolean gst_gl_mixer_propose_allocation (GstGLBaseMixer * base_mix, GstGLBaseMixerPad * base_pad, GstQuery * decide_query, GstQuery * query) @@ -366,7 +386,7 @@ gst_gl_mixer_class_init (GstGLMixerClass * klass) videoaggregator_class->get_output_buffer = gst_gl_mixer_get_output_buffer; videoaggregator_class->negotiated_caps = _negotiated_caps; videoaggregator_class->update_caps = _update_caps; - videoaggregator_class->find_best_format = NULL; + videoaggregator_class->find_best_format = _find_best_format; mix_class->propose_allocation = gst_gl_mixer_propose_allocation; mix_class->decide_allocation = gst_gl_mixer_decide_allocation; From 2362ee8bee735fbf1bda6d1639cabc60b5475f29 Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Wed, 13 May 2015 17:10:42 +1000 Subject: [PATCH 168/381] glvideomixer: don't upload the vertex data every frame Add the missing cache tracking statement. --- ext/gl/gstglvideomixer.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ext/gl/gstglvideomixer.c b/ext/gl/gstglvideomixer.c index cbba72a5d7..fe682074f8 100644 --- a/ext/gl/gstglvideomixer.c +++ b/ext/gl/gstglvideomixer.c @@ -954,6 +954,8 @@ gst_gl_video_mixer_callback (gpointer stuff) gl->BufferData (GL_ARRAY_BUFFER, 4 * 5 * sizeof (GLfloat), v_vertices, GL_STATIC_DRAW); + + pad->geometry_change = FALSE; } else { gl->BindBuffer (GL_ARRAY_BUFFER, pad->vertex_buffer); } From 7b48ab14e1368a220bef111faaff4530205e2393 Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Wed, 13 May 2015 17:11:55 +1000 Subject: [PATCH 169/381] glvideomixer: implement par handling We were previously ignoring it completely --- ext/gl/gstglvideomixer.c | 66 ++++++++++++++++++++++++++++++---------- 1 file changed, 50 insertions(+), 16 deletions(-) diff --git a/ext/gl/gstglvideomixer.c b/ext/gl/gstglvideomixer.c index fe682074f8..f7a3eb1fad 100644 --- a/ext/gl/gstglvideomixer.c +++ b/ext/gl/gstglvideomixer.c @@ -621,9 +621,49 @@ gst_gl_video_mixer_get_property (GObject * object, guint prop_id, } } +static void +_mixer_pad_get_output_size (GstGLVideoMixer * mix, + GstGLVideoMixerPad * mix_pad, gint * width, gint * height) +{ + GstVideoAggregator *vagg = GST_VIDEO_AGGREGATOR (mix); + GstVideoAggregatorPad *vagg_pad = GST_VIDEO_AGGREGATOR_PAD (mix_pad); + gint pad_width, pad_height; + gint dar_n, dar_d; + + pad_width = + mix_pad->width <= + 0 ? GST_VIDEO_INFO_WIDTH (&vagg_pad->info) : mix_pad->width; + pad_height = + mix_pad->height <= + 0 ? GST_VIDEO_INFO_HEIGHT (&vagg_pad->info) : mix_pad->height; + + gst_util_fraction_multiply (GST_VIDEO_INFO_PAR_N (&vagg_pad->info), + GST_VIDEO_INFO_PAR_D (&vagg_pad->info), + GST_VIDEO_INFO_PAR_D (&vagg->info), GST_VIDEO_INFO_PAR_N (&vagg->info), + &dar_n, &dar_d); + GST_LOG_OBJECT (mix_pad, "scaling %ux%u by %u/%u (%u/%u / %u/%u)", pad_width, + pad_height, dar_n, dar_d, GST_VIDEO_INFO_PAR_N (&vagg_pad->info), + GST_VIDEO_INFO_PAR_D (&vagg_pad->info), + GST_VIDEO_INFO_PAR_N (&vagg->info), GST_VIDEO_INFO_PAR_D (&vagg->info)); + + if (pad_height % dar_n == 0) { + pad_height = gst_util_uint64_scale_int (pad_width, dar_n, dar_d); + } else if (pad_width % dar_d == 0) { + pad_width = gst_util_uint64_scale_int (pad_height, dar_d, dar_n); + } else { + pad_height = gst_util_uint64_scale_int (pad_width, dar_n, dar_d); + } + + if (width) + *width = pad_width; + if (height) + *height = pad_height; +} + static GstCaps * _update_caps (GstVideoAggregator * vagg, GstCaps * caps) { + GstGLVideoMixer *mix = GST_GL_VIDEO_MIXER (vagg); GList *l; gint best_width = -1, best_height = -1; GstVideoInfo info; @@ -640,15 +680,7 @@ _update_caps (GstVideoAggregator * vagg, GstCaps * caps) gint this_width, this_height; gint width, height; - if (mixer_pad->width > 0) - width = mixer_pad->width; - else - width = GST_VIDEO_INFO_WIDTH (&vaggpad->info); - - if (mixer_pad->height > 0) - height = mixer_pad->height; - else - height = GST_VIDEO_INFO_HEIGHT (&vaggpad->info); + _mixer_pad_get_output_size (mix, mixer_pad, &width, &height); if (width == 0 || height == 0) continue; @@ -844,6 +876,7 @@ static void gst_gl_video_mixer_callback (gpointer stuff) { GstGLVideoMixer *video_mixer = GST_GL_VIDEO_MIXER (stuff); + GstVideoAggregator *vagg = GST_VIDEO_AGGREGATOR (stuff); GstGLMixer *mixer = GST_GL_MIXER (video_mixer); GstGLFuncs *gl = GST_GL_BASE_MIXER (mixer)->context->gl_vtable; @@ -858,8 +891,8 @@ gst_gl_video_mixer_callback (gpointer stuff) guint count = 0; - out_width = GST_VIDEO_INFO_WIDTH (&GST_VIDEO_AGGREGATOR (stuff)->info); - out_height = GST_VIDEO_INFO_HEIGHT (&GST_VIDEO_AGGREGATOR (stuff)->info); + out_width = GST_VIDEO_INFO_WIDTH (&vagg->info); + out_height = GST_VIDEO_INFO_HEIGHT (&vagg->info); gst_gl_context_clear_shader (GST_GL_BASE_MIXER (mixer)->context); gl->BindTexture (GL_TEXTURE_2D, 0); @@ -891,6 +924,7 @@ gst_gl_video_mixer_callback (gpointer stuff) while (count < video_mixer->input_frames->len) { GstGLMixerFrameData *frame; GstGLVideoMixerPad *pad; + GstVideoInfo *v_info; guint in_tex; guint in_width, in_height; @@ -910,8 +944,9 @@ gst_gl_video_mixer_callback (gpointer stuff) continue; } pad = (GstGLVideoMixerPad *) frame->pad; - in_width = GST_VIDEO_INFO_WIDTH (&GST_VIDEO_AGGREGATOR_PAD (pad)->info); - in_height = GST_VIDEO_INFO_HEIGHT (&GST_VIDEO_AGGREGATOR_PAD (pad)->info); + v_info = &GST_VIDEO_AGGREGATOR_PAD (pad)->info; + in_width = GST_VIDEO_INFO_WIDTH (v_info); + in_height = GST_VIDEO_INFO_HEIGHT (v_info); if (!frame->texture || in_width <= 0 || in_height <= 0 || pad->alpha == 0.0f) { @@ -924,11 +959,10 @@ gst_gl_video_mixer_callback (gpointer stuff) in_tex = frame->texture; if (pad->geometry_change || !pad->vertex_buffer) { - guint pad_width, pad_height; + gint pad_width, pad_height; gfloat w, h; - pad_width = pad->width <= 0 ? in_width : pad->width; - pad_height = pad->height <= 0 ? in_height : pad->height; + _mixer_pad_get_output_size (video_mixer, pad, &pad_width, &pad_height); w = ((gfloat) pad_width / (gfloat) out_width); h = ((gfloat) pad_height / (gfloat) out_height); From df7370bc2a63c40867d01428223cb1b08a456f44 Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Wed, 13 May 2015 17:38:35 +1000 Subject: [PATCH 170/381] compositor: fix rectangle obscure test to clamp against the output frame size Rather than one of the input pad video info's. The test checking this was not constraining the output frame size to ensure that the out of frame stream was not being displayed. --- gst/compositor/compositor.c | 14 ++++----- tests/check/elements/compositor.c | 50 +++++++++++++++++++++++-------- 2 files changed, 43 insertions(+), 21 deletions(-) diff --git a/gst/compositor/compositor.c b/gst/compositor/compositor.c index 756df1ac56..a97072948c 100644 --- a/gst/compositor/compositor.c +++ b/gst/compositor/compositor.c @@ -417,15 +417,13 @@ gst_compositor_pad_prepare_frame (GstVideoAggregatorPad * pad, * the case where (say, with negative xpos/ypos) the non-obscured portion of * the frame could be outside the bounds of the video itself and hence not * visible at all */ - frame_rect.x = CLAMP (cpad->xpos, 0, - GST_VIDEO_INFO_WIDTH (&pad->buffer_vinfo)); - frame_rect.y = CLAMP (cpad->ypos, 0, - GST_VIDEO_INFO_HEIGHT (&pad->buffer_vinfo)); + frame_rect.x = CLAMP (cpad->xpos, 0, GST_VIDEO_INFO_WIDTH (&vagg->info)); + frame_rect.y = CLAMP (cpad->ypos, 0, GST_VIDEO_INFO_HEIGHT (&vagg->info)); /* Clamp the width/height to the frame boundaries as well */ - frame_rect.w = MIN (GST_VIDEO_INFO_WIDTH (&cpad->conversion_info), - GST_VIDEO_INFO_WIDTH (&pad->buffer_vinfo) - cpad->xpos); - frame_rect.h = MIN (GST_VIDEO_INFO_HEIGHT (&cpad->conversion_info), - GST_VIDEO_INFO_HEIGHT (&pad->buffer_vinfo) - cpad->ypos); + frame_rect.w = + MAX (GST_VIDEO_INFO_WIDTH (&cpad->conversion_info) - frame_rect.x, 0); + frame_rect.h = + MAX (GST_VIDEO_INFO_HEIGHT (&cpad->conversion_info) - frame_rect.y, 0); GST_OBJECT_LOCK (vagg); /* Check if this frame is obscured by a higher-zorder frame diff --git a/tests/check/elements/compositor.c b/tests/check/elements/compositor.c index 4ff890b66b..7ce637eb00 100644 --- a/tests/check/elements/compositor.c +++ b/tests/check/elements/compositor.c @@ -1138,9 +1138,10 @@ test_obscured_pad_probe_cb (GstPad * srcpad, GstPadProbeInfo * info, static void _test_obscured (const gchar * caps_str, gint xpos0, gint ypos0, gint width0, gint height0, gdouble alpha0, gint xpos1, gint ypos1, gint width1, - gint height1, gdouble alpha1) + gint height1, gdouble alpha1, gint out_width, gint out_height) { GstElement *pipeline, *sink, *mix, *src0, *cfilter0, *src1, *cfilter1; + GstElement *out_cfilter; GstPad *srcpad, *sinkpad; GstSample *last_sample = NULL; GstSample *sample; @@ -1164,12 +1165,22 @@ _test_obscured (const gchar * caps_str, gint xpos0, gint ypos0, gint width0, gst_caps_unref (caps); mix = gst_element_factory_make ("compositor", "compositor"); + out_cfilter = gst_element_factory_make ("capsfilter", "out_capsfilter"); + caps = gst_caps_from_string (caps_str); + if (out_width > 0) + gst_caps_set_simple (caps, "width", G_TYPE_INT, out_width, NULL); + if (out_height > 0) + gst_caps_set_simple (caps, "height", G_TYPE_INT, out_height, NULL); + g_object_set (out_cfilter, "caps", caps, NULL); + gst_caps_unref (caps); sink = gst_element_factory_make ("appsink", "sink"); + gst_bin_add_many (GST_BIN (pipeline), src0, cfilter0, src1, cfilter1, mix, - sink, NULL); + out_cfilter, sink, NULL); fail_unless (gst_element_link (src0, cfilter0)); fail_unless (gst_element_link (src1, cfilter1)); - fail_unless (gst_element_link (mix, sink)); + fail_unless (gst_element_link (mix, out_cfilter)); + fail_unless (gst_element_link (out_cfilter, sink)); srcpad = gst_element_get_static_pad (cfilter0, "src"); sinkpad = gst_element_get_request_pad (mix, "sink_0"); @@ -1214,6 +1225,7 @@ GST_START_TEST (test_obscured_skipped) gint ypos0, ypos1; gint width0, width1; gint height0, height1; + gint out_width, out_height; gdouble alpha0, alpha1; const gchar *caps_str; @@ -1223,20 +1235,21 @@ GST_START_TEST (test_obscured_skipped) alpha0 = alpha1 = 1.0; xpos0 = xpos1 = ypos0 = ypos1 = 0; width0 = width1 = height0 = height1 = 0; + out_width = out_height = 0; GST_INFO ("testing defaults"); /* With everything at defaults, sink_1 will obscure sink_0, so buffers from * sink_0 will never get mapped by compositor. To verify, run with * GST_DEBUG=compositor:6 and look for "Obscured by" messages */ _test_obscured (caps_str, xpos0, ypos0, width0, height0, alpha0, xpos1, ypos1, - width1, height1, alpha1); + width1, height1, alpha1, out_width, out_height); fail_unless (buffer_mapped == FALSE); buffer_mapped = FALSE; caps_str = "video/x-raw,format=ARGB"; GST_INFO ("testing video with alpha channel"); _test_obscured (caps_str, xpos0, ypos0, width0, height0, alpha0, xpos1, ypos1, - width1, height1, alpha1); + width1, height1, alpha1, out_width, out_height); fail_unless (buffer_mapped == TRUE); caps_str = "video/x-raw"; buffer_mapped = FALSE; @@ -1244,7 +1257,7 @@ GST_START_TEST (test_obscured_skipped) alpha1 = 0.0; GST_INFO ("testing alpha1 = %.2g", alpha1); _test_obscured (caps_str, xpos0, ypos0, width0, height0, alpha0, xpos1, ypos1, - width1, height1, alpha1); + width1, height1, alpha1, out_width, out_height); fail_unless (buffer_mapped == TRUE); alpha1 = 1.0; buffer_mapped = FALSE; @@ -1253,7 +1266,7 @@ GST_START_TEST (test_obscured_skipped) for (alpha1 = 1; alpha1 < 10; alpha1 += 1) { GST_INFO ("testing alpha1 = %.2g", alpha1 / 10); _test_obscured (caps_str, xpos0, ypos0, width0, height0, alpha0, xpos1, - ypos1, width1, height1, alpha1 / 10); + ypos1, width1, height1, alpha1 / 10, out_width, out_height); fail_unless (buffer_mapped == TRUE); } alpha1 = 1.0; @@ -1262,7 +1275,7 @@ GST_START_TEST (test_obscured_skipped) width1 = height1 = 10; GST_INFO ("testing smaller sink_1"); _test_obscured (caps_str, xpos0, ypos0, width0, height0, alpha0, xpos1, ypos1, - width1, height1, alpha1); + width1, height1, alpha1, out_width, out_height); fail_unless (buffer_mapped == TRUE); width1 = height1 = 0; buffer_mapped = FALSE; @@ -1270,7 +1283,7 @@ GST_START_TEST (test_obscured_skipped) width0 = height0 = width1 = height1 = 10; GST_INFO ("testing smaller sink_1 and sink0 (same sizes)"); _test_obscured (caps_str, xpos0, ypos0, width0, height0, alpha0, xpos1, ypos1, - width1, height1, alpha1); + width1, height1, alpha1, out_width, out_height); fail_unless (buffer_mapped == FALSE); width0 = height0 = width1 = height1 = 0; buffer_mapped = FALSE; @@ -1279,7 +1292,7 @@ GST_START_TEST (test_obscured_skipped) width1 = height1 = 10; GST_INFO ("testing smaller sink_1 and sink0 (sink_0 > sink_1)"); _test_obscured (caps_str, xpos0, ypos0, width0, height0, alpha0, xpos1, ypos1, - width1, height1, alpha1); + width1, height1, alpha1, out_width, out_height); fail_unless (buffer_mapped == TRUE); width0 = height0 = width1 = height1 = 0; buffer_mapped = FALSE; @@ -1288,17 +1301,28 @@ GST_START_TEST (test_obscured_skipped) width1 = height1 = 20; GST_INFO ("testing smaller sink_1 and sink0 (sink_0 < sink_1)"); _test_obscured (caps_str, xpos0, ypos0, width0, height0, alpha0, xpos1, ypos1, - width1, height1, alpha1); + width1, height1, alpha1, out_width, out_height); fail_unless (buffer_mapped == FALSE); width0 = height0 = width1 = height1 = 0; buffer_mapped = FALSE; + xpos0 = ypos0 = 10; + xpos1 = ypos1 = 20; + GST_INFO ("testing offset"); + _test_obscured (caps_str, xpos0, ypos0, width0, height0, alpha0, xpos1, ypos1, + width1, height1, alpha1, out_width, out_height); + fail_unless (buffer_mapped == TRUE); + xpos0 = ypos0 = xpos1 = ypos1 = 0; + buffer_mapped = FALSE; + xpos0 = ypos0 = 10000; + out_width = 320; + out_height = 240; GST_INFO ("testing sink_0 outside the frame"); _test_obscured (caps_str, xpos0, ypos0, width0, height0, alpha0, xpos1, ypos1, - width1, height1, alpha1); + width1, height1, alpha1, out_width, out_height); fail_unless (buffer_mapped == FALSE); - xpos0 = ypos0 = 0; + xpos0 = ypos0 = out_width = out_height = 0; buffer_mapped = FALSE; } From e6859acff2ffcebbbe7fb120193067487d110850 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Wed, 13 May 2015 15:42:15 +0300 Subject: [PATCH 171/381] glmixerbin: Don't unref pad templates Otherwise we unref the reference that is owned by the element class. --- ext/gl/gstglmixerbin.c | 1 - 1 file changed, 1 deletion(-) diff --git a/ext/gl/gstglmixerbin.c b/ext/gl/gstglmixerbin.c index 47fff2a542..ae8abef95a 100644 --- a/ext/gl/gstglmixerbin.c +++ b/ext/gl/gstglmixerbin.c @@ -424,7 +424,6 @@ gst_gl_mixer_bin_request_new_pad (GstElement * element, GstPadTemplate * templ, mixer_pad = gst_element_request_pad (self->mixer, mixer_templ, req_name, NULL); - gst_object_unref (mixer_templ); g_return_val_if_fail (mixer_pad, NULL); if (!_create_input_chain (self, chain, mixer_pad)) { From d3d74988781b5d5510d2a04d148eec569ae8e300 Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Wed, 13 May 2015 23:54:52 +1000 Subject: [PATCH 172/381] gl: don't deadlock on context creation failure https://bugzilla.gnome.org/show_bug.cgi?id=749284 --- ext/gl/gstglbasemixer.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ext/gl/gstglbasemixer.c b/ext/gl/gstglbasemixer.c index 0e670e044d..d462c161ad 100644 --- a/ext/gl/gstglbasemixer.c +++ b/ext/gl/gstglbasemixer.c @@ -477,8 +477,10 @@ gst_gl_base_mixer_decide_allocation (GstGLBaseMixer * mix, GstQuery * query) if (!mix->context) { mix->context = gst_gl_context_new (mix->display); if (!gst_gl_context_create (mix->context, mix->priv->other_context, - &error)) + &error)) { + GST_OBJECT_UNLOCK (mix->display); goto context_error; + } } } while (!gst_gl_display_add_context (mix->display, mix->context)); GST_OBJECT_UNLOCK (mix->display); From b2eea8873cc2097c97520c6261c305d3797d20fc Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Thu, 14 May 2015 13:04:21 +1000 Subject: [PATCH 173/381] compositor: implement proper par handling We were previously failing on different input and output par --- gst-libs/gst/video/gstvideoaggregator.c | 8 +- gst/compositor/compositor.c | 107 +++++++++++++++--------- 2 files changed, 71 insertions(+), 44 deletions(-) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index 965aafcd43..d31690fcf1 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -748,6 +748,8 @@ gst_videoaggregator_update_src_caps (GstVideoAggregator * vagg) gst_structure_fixate_field_nearest_int (s, "height", best_height); gst_structure_fixate_field_nearest_fraction (s, "framerate", best_fps_n, best_fps_d); + gst_structure_fixate_field_nearest_fraction (s, "pixel-aspect-ratio", 1, + 1); /* fixate the the rest of the fields */ caps = gst_caps_fixate (caps); @@ -848,9 +850,6 @@ gst_videoaggregator_pad_sink_getcaps (GstPad * pad, GstVideoAggregator * vagg, gst_structure_set (s, "width", GST_TYPE_INT_RANGE, 1, G_MAXINT, "height", GST_TYPE_INT_RANGE, 1, G_MAXINT, "framerate", GST_TYPE_FRACTION_RANGE, 0, 1, G_MAXINT, 1, NULL); - if (!gst_structure_has_field (s, "pixel-aspect-ratio")) - gst_structure_set (s, "pixel-aspect-ratio", GST_TYPE_FRACTION, 1, 1, - NULL); gst_structure_remove_fields (s, "colorimetry", "chroma-site", "format", NULL); @@ -1842,9 +1841,6 @@ gst_videoaggregator_pad_sink_acceptcaps (GstPad * pad, gst_structure_set (s, "width", GST_TYPE_INT_RANGE, 1, G_MAXINT, "height", GST_TYPE_INT_RANGE, 1, G_MAXINT, "framerate", GST_TYPE_FRACTION_RANGE, 0, 1, G_MAXINT, 1, NULL); - if (!gst_structure_has_field (s, "pixel-aspect-ratio")) - gst_structure_set (s, "pixel-aspect-ratio", GST_TYPE_FRACTION, 1, 1, - NULL); gst_structure_remove_fields (s, "colorimetry", "chroma-site", "format", NULL); diff --git a/gst/compositor/compositor.c b/gst/compositor/compositor.c index a97072948c..942c48036e 100644 --- a/gst/compositor/compositor.c +++ b/gst/compositor/compositor.c @@ -214,11 +214,56 @@ gst_compositor_pad_set_property (GObject * object, guint prop_id, } } +static void +_mixer_pad_get_output_size (GstCompositor * comp, + GstCompositorPad * comp_pad, gint * width, gint * height) +{ + GstVideoAggregator *vagg = GST_VIDEO_AGGREGATOR (comp); + GstVideoAggregatorPad *vagg_pad = GST_VIDEO_AGGREGATOR_PAD (comp_pad); + gint pad_width, pad_height; + gint dar_n, dar_d; + + pad_width = + comp_pad->width <= + 0 ? GST_VIDEO_INFO_WIDTH (&vagg_pad->info) : comp_pad->width; + pad_height = + comp_pad->height <= + 0 ? GST_VIDEO_INFO_HEIGHT (&vagg_pad->info) : comp_pad->height; + + gst_util_fraction_multiply (GST_VIDEO_INFO_PAR_N (&vagg_pad->info), + GST_VIDEO_INFO_PAR_D (&vagg_pad->info), + GST_VIDEO_INFO_PAR_D (&vagg->info), GST_VIDEO_INFO_PAR_N (&vagg->info), + &dar_n, &dar_d); + GST_LOG_OBJECT (comp_pad, "scaling %ux%u by %u/%u (%u/%u / %u/%u)", pad_width, + pad_height, dar_n, dar_d, GST_VIDEO_INFO_PAR_N (&vagg_pad->info), + GST_VIDEO_INFO_PAR_D (&vagg_pad->info), + GST_VIDEO_INFO_PAR_N (&vagg->info), GST_VIDEO_INFO_PAR_D (&vagg->info)); + + if (pad_height % dar_n == 0) { + pad_height = gst_util_uint64_scale_int (pad_width, dar_n, dar_d); + } else if (pad_width % dar_d == 0) { + pad_width = gst_util_uint64_scale_int (pad_height, dar_d, dar_n); + } else { + pad_height = gst_util_uint64_scale_int (pad_width, dar_n, dar_d); + } + + if (width) + *width = pad_width; + if (height) + *height = pad_height; + + if (width) + *width = pad_width; + if (height) + *height = pad_height; +} + static gboolean gst_compositor_pad_set_info (GstVideoAggregatorPad * pad, GstVideoAggregator * vagg G_GNUC_UNUSED, GstVideoInfo * current_info, GstVideoInfo * wanted_info) { + GstCompositor *comp = GST_COMPOSITOR (vagg); GstCompositorPad *cpad = GST_COMPOSITOR_PAD (pad); gchar *colorimetry, *best_colorimetry; const gchar *chroma, *best_chroma; @@ -242,15 +287,7 @@ gst_compositor_pad_set_info (GstVideoAggregatorPad * pad, gst_video_colorimetry_to_string (&(wanted_info->colorimetry)); best_chroma = gst_video_chroma_to_string (wanted_info->chroma_site); - if (cpad->width > 0) - width = cpad->width; - else - width = current_info->width; - - if (cpad->height > 0) - height = cpad->height; - else - height = current_info->height; + _mixer_pad_get_output_size (comp, cpad, &width, &height); if (GST_VIDEO_INFO_FORMAT (wanted_info) != GST_VIDEO_INFO_FORMAT (current_info) @@ -268,8 +305,8 @@ gst_compositor_pad_set_info (GstVideoAggregatorPad * pad, width, height); tmp_info.chroma_site = wanted_info->chroma_site; tmp_info.colorimetry = wanted_info->colorimetry; - tmp_info.par_n = current_info->par_n; - tmp_info.par_d = current_info->par_d; + tmp_info.par_n = vagg->info.par_n; + tmp_info.par_d = vagg->info.par_d; tmp_info.fps_n = current_info->fps_n; tmp_info.fps_d = current_info->fps_d; tmp_info.flags = current_info->flags; @@ -278,6 +315,7 @@ gst_compositor_pad_set_info (GstVideoAggregatorPad * pad, GST_DEBUG_OBJECT (pad, "This pad will be converted from %d to %d", GST_VIDEO_INFO_FORMAT (current_info), GST_VIDEO_INFO_FORMAT (&tmp_info)); + cpad->convert = gst_video_converter_new (current_info, &tmp_info, NULL); cpad->conversion_info = tmp_info; if (!cpad->convert) { @@ -311,6 +349,7 @@ static gboolean gst_compositor_pad_prepare_frame (GstVideoAggregatorPad * pad, GstVideoAggregator * vagg) { + GstCompositor *comp = GST_COMPOSITOR (vagg); GstCompositorPad *cpad = GST_COMPOSITOR_PAD (pad); guint outsize; GstVideoFrame *converted_frame; @@ -338,15 +377,8 @@ gst_compositor_pad_prepare_frame (GstVideoAggregatorPad * pad, * Equal to cpad->width/height if it's set, otherwise it's the pad * width/height. See ->set_info() * */ - if (cpad->width > 0) - width = cpad->width; - else - width = GST_VIDEO_INFO_WIDTH (&pad->buffer_vinfo); - if (cpad->height > 0) - height = cpad->height; - else - height = GST_VIDEO_INFO_HEIGHT (&pad->buffer_vinfo); + _mixer_pad_get_output_size (comp, cpad, &width, &height); /* The only thing that can change here is the width * and height, otherwise set_info would've been called */ @@ -384,8 +416,8 @@ gst_compositor_pad_prepare_frame (GstVideoAggregatorPad * pad, width, height); tmp_info.chroma_site = cpad->conversion_info.chroma_site; tmp_info.colorimetry = cpad->conversion_info.colorimetry; - tmp_info.par_n = cpad->conversion_info.par_n; - tmp_info.par_d = cpad->conversion_info.par_d; + tmp_info.par_n = vagg->info.par_n; + tmp_info.par_d = vagg->info.par_d; tmp_info.fps_n = cpad->conversion_info.fps_n; tmp_info.fps_d = cpad->conversion_info.fps_d; tmp_info.flags = cpad->conversion_info.flags; @@ -394,6 +426,7 @@ gst_compositor_pad_prepare_frame (GstVideoAggregatorPad * pad, GST_DEBUG_OBJECT (pad, "This pad will be converted from %d to %d", GST_VIDEO_INFO_FORMAT (&pad->buffer_vinfo), GST_VIDEO_INFO_FORMAT (&tmp_info)); + cpad->convert = gst_video_converter_new (&pad->buffer_vinfo, &tmp_info, NULL); cpad->conversion_info = tmp_info; @@ -420,10 +453,8 @@ gst_compositor_pad_prepare_frame (GstVideoAggregatorPad * pad, frame_rect.x = CLAMP (cpad->xpos, 0, GST_VIDEO_INFO_WIDTH (&vagg->info)); frame_rect.y = CLAMP (cpad->ypos, 0, GST_VIDEO_INFO_HEIGHT (&vagg->info)); /* Clamp the width/height to the frame boundaries as well */ - frame_rect.w = - MAX (GST_VIDEO_INFO_WIDTH (&cpad->conversion_info) - frame_rect.x, 0); - frame_rect.h = - MAX (GST_VIDEO_INFO_HEIGHT (&cpad->conversion_info) - frame_rect.y, 0); + frame_rect.w = MAX (width - frame_rect.x, 0); + frame_rect.h = MAX (height - frame_rect.y, 0); GST_OBJECT_LOCK (vagg); /* Check if this frame is obscured by a higher-zorder frame @@ -434,16 +465,17 @@ gst_compositor_pad_prepare_frame (GstVideoAggregatorPad * pad, GstVideoRectangle frame2_rect; GstVideoAggregatorPad *pad2 = l->data; GstCompositorPad *cpad2 = GST_COMPOSITOR_PAD (pad2); + gint pad2_width, pad2_height; + + _mixer_pad_get_output_size (comp, cpad2, &pad2_width, &pad2_height); /* We don't need to clamp the coords of the second rectangle */ frame2_rect.x = cpad2->xpos; frame2_rect.y = cpad2->ypos; /* This is effectively what set_info and the above conversion * code do to calculate the desired width/height */ - frame2_rect.w = cpad2->width ? cpad2->width : - GST_VIDEO_INFO_WIDTH (&cpad2->conversion_info); - frame2_rect.h = cpad2->height ? cpad2->height : - GST_VIDEO_INFO_HEIGHT (&cpad2->conversion_info); + frame2_rect.w = pad2_width; + frame2_rect.h = pad2_height; /* Check if there's a buffer to be aggregated, ensure it can't have an alpha * channel, then check opacity and frame boundaries */ @@ -831,6 +863,9 @@ _update_caps (GstVideoAggregator * vagg, GstCaps * caps) gst_video_info_from_caps (&info, caps); + /* FIXME: this doesn't work for non 1/1 output par's as we don't have that + * information available at this time */ + GST_OBJECT_LOCK (vagg); for (l = GST_ELEMENT (vagg)->sinkpads; l; l = l->next) { GstVideoAggregatorPad *vaggpad = l->data; @@ -838,15 +873,8 @@ _update_caps (GstVideoAggregator * vagg, GstCaps * caps) gint this_width, this_height; gint width, height; - if (compositor_pad->width > 0) - width = compositor_pad->width; - else - width = GST_VIDEO_INFO_WIDTH (&vaggpad->info); - - if (compositor_pad->height > 0) - height = compositor_pad->height; - else - height = GST_VIDEO_INFO_HEIGHT (&vaggpad->info); + _mixer_pad_get_output_size (GST_COMPOSITOR (vagg), compositor_pad, &width, + &height); if (width == 0 || height == 0) continue; @@ -866,6 +894,9 @@ _update_caps (GstVideoAggregator * vagg, GstCaps * caps) info.height = best_height; if (set_functions (GST_COMPOSITOR (vagg), &info)) ret = gst_video_info_to_caps (&info); + + gst_caps_set_simple (ret, "pixel-aspect-ratio", GST_TYPE_FRACTION_RANGE, + 1, G_MAXINT, G_MAXINT, 1, NULL); } return ret; From 930742dd5a2113ff10e5625db1f9ee25893e1a08 Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Fri, 1 May 2015 12:04:28 +1000 Subject: [PATCH 174/381] gl: element buffers are part of vao state Use them as such. They are also required for GL3 core profile support with glDrawElements on OS X. --- ext/gl/gstglvideomixer.c | 41 +++++++++++++++++++++++++++++----------- ext/gl/gstglvideomixer.h | 1 + 2 files changed, 31 insertions(+), 11 deletions(-) diff --git a/ext/gl/gstglvideomixer.c b/ext/gl/gstglvideomixer.c index f7a3eb1fad..f39313772d 100644 --- a/ext/gl/gstglvideomixer.c +++ b/ext/gl/gstglvideomixer.c @@ -733,6 +733,11 @@ _reset_gl (GstGLContext * context, GstGLVideoMixer * video_mixer) video_mixer->vao = 0; } + if (video_mixer->vbo_indices) { + gl->DeleteBuffers (1, &video_mixer->vbo_indices); + video_mixer->vbo_indices = 0; + } + gst_aggregator_iterate_sinkpads (GST_AGGREGATOR (video_mixer), _reset_pad_gl, NULL); } @@ -790,6 +795,21 @@ gst_gl_video_mixer_process_textures (GstGLMixer * mix, GPtrArray * frames, return TRUE; } +static const GLushort indices[] = { 0, 1, 2, 0, 2, 3 }; + +static void +_init_vbo_indices (GstGLVideoMixer * mixer) +{ + const GstGLFuncs *gl = GST_GL_BASE_MIXER (mixer)->context->gl_vtable; + + if (!mixer->vbo_indices) { + gl->GenBuffers (1, &mixer->vbo_indices); + gl->BindBuffer (GL_ELEMENT_ARRAY_BUFFER, mixer->vbo_indices); + gl->BufferData (GL_ELEMENT_ARRAY_BUFFER, sizeof (indices), indices, + GL_STATIC_DRAW); + } +} + static gboolean _draw_checker_background (GstGLVideoMixer * video_mixer) { @@ -797,10 +817,6 @@ _draw_checker_background (GstGLVideoMixer * video_mixer) const GstGLFuncs *gl = GST_GL_BASE_MIXER (mixer)->context->gl_vtable; gint attr_position_loc = 0; - const GLushort indices[] = { - 0, 1, 2, - 0, 2, 3 - }; /* *INDENT-OFF* */ gfloat v_vertices[] = { -1.0,-1.0,-1.0f, @@ -820,12 +836,15 @@ _draw_checker_background (GstGLVideoMixer * video_mixer) attr_position_loc = gst_gl_shader_get_attribute_location (video_mixer->checker, "a_position"); + _init_vbo_indices (video_mixer); + if (!video_mixer->checker_vbo) { gl->GenBuffers (1, &video_mixer->checker_vbo); gl->BindBuffer (GL_ARRAY_BUFFER, video_mixer->checker_vbo); gl->BufferData (GL_ARRAY_BUFFER, 4 * 3 * sizeof (GLfloat), v_vertices, GL_STATIC_DRAW); } else { + gl->BindBuffer (GL_ELEMENT_ARRAY_BUFFER, video_mixer->vbo_indices); gl->BindBuffer (GL_ARRAY_BUFFER, video_mixer->checker_vbo); } @@ -834,9 +853,10 @@ _draw_checker_background (GstGLVideoMixer * video_mixer) gl->EnableVertexAttribArray (attr_position_loc); - gl->DrawElements (GL_TRIANGLES, 6, GL_UNSIGNED_SHORT, indices); + gl->DrawElements (GL_TRIANGLES, 6, GL_UNSIGNED_SHORT, 0); gl->DisableVertexAttribArray (attr_position_loc); + gl->BindBuffer (GL_ELEMENT_ARRAY_BUFFER, 0); gl->BindBuffer (GL_ARRAY_BUFFER, 0); return TRUE; @@ -884,11 +904,6 @@ gst_gl_video_mixer_callback (gpointer stuff) GLint attr_texture_loc = 0; guint out_width, out_height; - const GLushort indices[] = { - 0, 1, 2, - 0, 2, 3 - }; - guint count = 0; out_width = GST_VIDEO_INFO_WIDTH (&vagg->info); @@ -958,6 +973,8 @@ gst_gl_video_mixer_callback (gpointer stuff) in_tex = frame->texture; + _init_vbo_indices (video_mixer); + if (pad->geometry_change || !pad->vertex_buffer) { gint pad_width, pad_height; gfloat w, h; @@ -993,6 +1010,7 @@ gst_gl_video_mixer_callback (gpointer stuff) } else { gl->BindBuffer (GL_ARRAY_BUFFER, pad->vertex_buffer); } + gl->BindBuffer (GL_ELEMENT_ARRAY_BUFFER, video_mixer->vbo_indices); gl->BlendFunc (GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA); gl->BlendEquation (GL_FUNC_ADD); @@ -1011,7 +1029,7 @@ gst_gl_video_mixer_callback (gpointer stuff) gl->VertexAttribPointer (attr_texture_loc, 2, GL_FLOAT, GL_FALSE, 5 * sizeof (GLfloat), (void *) (3 * sizeof (GLfloat))); - gl->DrawElements (GL_TRIANGLES, 6, GL_UNSIGNED_SHORT, indices); + gl->DrawElements (GL_TRIANGLES, 6, GL_UNSIGNED_SHORT, 0); ++count; } @@ -1022,6 +1040,7 @@ gst_gl_video_mixer_callback (gpointer stuff) if (gl->GenVertexArrays) gl->BindVertexArray (0); + gl->BindBuffer (GL_ELEMENT_ARRAY_BUFFER, 0); gl->BindBuffer (GL_ARRAY_BUFFER, 0); gl->BindTexture (GL_TEXTURE_2D, 0); diff --git a/ext/gl/gstglvideomixer.h b/ext/gl/gstglvideomixer.h index 967358baba..0d0252b56f 100644 --- a/ext/gl/gstglvideomixer.h +++ b/ext/gl/gstglvideomixer.h @@ -64,6 +64,7 @@ struct _GstGLVideoMixer GPtrArray *input_frames; GLuint vao; + GLuint vbo_indices; GLuint checker_vbo; }; From 7e5123cd90ce7a5009fcdea8103f4b0097fb3531 Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Thu, 14 May 2015 16:42:09 +1000 Subject: [PATCH 175/381] gl: remove useless gl{En,Dis}able (GL_TEXTURE_*) calls We are using shaders everywhere and so they are not needed --- ext/gl/gstglmosaic.c | 1 - ext/gl/gstglvideomixer.c | 3 --- 2 files changed, 4 deletions(-) diff --git a/ext/gl/gstglmosaic.c b/ext/gl/gstglmosaic.c index 726674b4df..0a9834138b 100644 --- a/ext/gl/gstglmosaic.c +++ b/ext/gl/gstglmosaic.c @@ -242,7 +242,6 @@ gst_gl_mosaic_callback (gpointer stuff) gst_gl_context_clear_shader (GST_GL_BASE_MIXER (mixer)->context); gl->BindTexture (GL_TEXTURE_2D, 0); - gl->Disable (GL_TEXTURE_2D); gl->Enable (GL_DEPTH_TEST); diff --git a/ext/gl/gstglvideomixer.c b/ext/gl/gstglvideomixer.c index f39313772d..d27396f4cf 100644 --- a/ext/gl/gstglvideomixer.c +++ b/ext/gl/gstglvideomixer.c @@ -911,9 +911,6 @@ gst_gl_video_mixer_callback (gpointer stuff) gst_gl_context_clear_shader (GST_GL_BASE_MIXER (mixer)->context); gl->BindTexture (GL_TEXTURE_2D, 0); - if (gst_gl_context_get_gl_api (GST_GL_BASE_MIXER (mixer)->context) & - GST_GL_API_OPENGL) - gl->Disable (GL_TEXTURE_2D); gl->Disable (GL_DEPTH_TEST); gl->Disable (GL_CULL_FACE); From c90bc869e5d11f142833fc09d94d6f22da3d9e26 Mon Sep 17 00:00:00 2001 From: Nicolas Dufresne Date: Sat, 16 May 2015 23:38:14 -0400 Subject: [PATCH 176/381] doc: Workaround gtkdoc issue With gtkdoc 1.22, the XML generator fails when a itemizedlist is followed by a refsect2. Workaround the issue by wrapping the refsect2 into para. --- gst/compositor/compositor.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gst/compositor/compositor.c b/gst/compositor/compositor.c index 942c48036e..7ab6c85377 100644 --- a/gst/compositor/compositor.c +++ b/gst/compositor/compositor.c @@ -61,6 +61,7 @@ * * * + * * * Sample pipelines * |[ @@ -104,6 +105,7 @@ * timeoverlay ! queue2 ! comp. * ]| A pipeline to demonstrate synchronized compositing (the second stream starts after 3 seconds) * + * */ #ifdef HAVE_CONFIG_H From 1bd590ab3bf6d468b95df48ffe85184be4be498f Mon Sep 17 00:00:00 2001 From: Stefan Sauer Date: Mon, 18 May 2015 20:16:32 +0200 Subject: [PATCH 177/381] Revert "doc: Workaround gtkdoc issue" This reverts commit ff6c736fe08e01f4320c4b02e811a0b57cf97cc1. This is fixed by the gtk-doc 1.23 release. cannot contain : http://www.docbook.org/tdg/en/html/para.html http://www.docbook.org/tdg/en/html/refsect2.html --- gst/compositor/compositor.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/gst/compositor/compositor.c b/gst/compositor/compositor.c index 7ab6c85377..942c48036e 100644 --- a/gst/compositor/compositor.c +++ b/gst/compositor/compositor.c @@ -61,7 +61,6 @@ * * * - * * * Sample pipelines * |[ @@ -105,7 +104,6 @@ * timeoverlay ! queue2 ! comp. * ]| A pipeline to demonstrate synchronized compositing (the second stream starts after 3 seconds) * - * */ #ifdef HAVE_CONFIG_H From 012e636e4ef433aba38e9793cba091359612fc7b Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Thu, 21 May 2015 00:56:01 +1000 Subject: [PATCH 178/381] compositor/glvideomixer: fix up par handling We were using the wrong formula https://bugzilla.gnome.org/show_bug.cgi?id=749634 --- ext/gl/gstglvideomixer.c | 15 ++++++++------- gst/compositor/compositor.c | 15 ++++++++------- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/ext/gl/gstglvideomixer.c b/ext/gl/gstglvideomixer.c index d27396f4cf..148c702610 100644 --- a/ext/gl/gstglvideomixer.c +++ b/ext/gl/gstglvideomixer.c @@ -628,7 +628,7 @@ _mixer_pad_get_output_size (GstGLVideoMixer * mix, GstVideoAggregator *vagg = GST_VIDEO_AGGREGATOR (mix); GstVideoAggregatorPad *vagg_pad = GST_VIDEO_AGGREGATOR_PAD (mix_pad); gint pad_width, pad_height; - gint dar_n, dar_d; + guint dar_n, dar_d; pad_width = mix_pad->width <= @@ -637,21 +637,22 @@ _mixer_pad_get_output_size (GstGLVideoMixer * mix, mix_pad->height <= 0 ? GST_VIDEO_INFO_HEIGHT (&vagg_pad->info) : mix_pad->height; - gst_util_fraction_multiply (GST_VIDEO_INFO_PAR_N (&vagg_pad->info), + gst_video_calculate_display_ratio (&dar_n, &dar_d, pad_width, pad_height, + GST_VIDEO_INFO_PAR_N (&vagg_pad->info), GST_VIDEO_INFO_PAR_D (&vagg_pad->info), - GST_VIDEO_INFO_PAR_D (&vagg->info), GST_VIDEO_INFO_PAR_N (&vagg->info), - &dar_n, &dar_d); + GST_VIDEO_INFO_PAR_N (&vagg->info), GST_VIDEO_INFO_PAR_D (&vagg->info) + ); GST_LOG_OBJECT (mix_pad, "scaling %ux%u by %u/%u (%u/%u / %u/%u)", pad_width, pad_height, dar_n, dar_d, GST_VIDEO_INFO_PAR_N (&vagg_pad->info), GST_VIDEO_INFO_PAR_D (&vagg_pad->info), GST_VIDEO_INFO_PAR_N (&vagg->info), GST_VIDEO_INFO_PAR_D (&vagg->info)); if (pad_height % dar_n == 0) { - pad_height = gst_util_uint64_scale_int (pad_width, dar_n, dar_d); + pad_width = gst_util_uint64_scale_int (pad_height, dar_n, dar_d); } else if (pad_width % dar_d == 0) { - pad_width = gst_util_uint64_scale_int (pad_height, dar_d, dar_n); + pad_height = gst_util_uint64_scale_int (pad_width, dar_d, dar_n); } else { - pad_height = gst_util_uint64_scale_int (pad_width, dar_n, dar_d); + pad_width = gst_util_uint64_scale_int (pad_height, dar_n, dar_d); } if (width) diff --git a/gst/compositor/compositor.c b/gst/compositor/compositor.c index 942c48036e..dd1ba7071d 100644 --- a/gst/compositor/compositor.c +++ b/gst/compositor/compositor.c @@ -221,7 +221,7 @@ _mixer_pad_get_output_size (GstCompositor * comp, GstVideoAggregator *vagg = GST_VIDEO_AGGREGATOR (comp); GstVideoAggregatorPad *vagg_pad = GST_VIDEO_AGGREGATOR_PAD (comp_pad); gint pad_width, pad_height; - gint dar_n, dar_d; + guint dar_n, dar_d; pad_width = comp_pad->width <= @@ -230,21 +230,22 @@ _mixer_pad_get_output_size (GstCompositor * comp, comp_pad->height <= 0 ? GST_VIDEO_INFO_HEIGHT (&vagg_pad->info) : comp_pad->height; - gst_util_fraction_multiply (GST_VIDEO_INFO_PAR_N (&vagg_pad->info), + gst_video_calculate_display_ratio (&dar_n, &dar_d, pad_width, pad_height, + GST_VIDEO_INFO_PAR_N (&vagg_pad->info), GST_VIDEO_INFO_PAR_D (&vagg_pad->info), - GST_VIDEO_INFO_PAR_D (&vagg->info), GST_VIDEO_INFO_PAR_N (&vagg->info), - &dar_n, &dar_d); + GST_VIDEO_INFO_PAR_N (&vagg->info), GST_VIDEO_INFO_PAR_D (&vagg->info) + ); GST_LOG_OBJECT (comp_pad, "scaling %ux%u by %u/%u (%u/%u / %u/%u)", pad_width, pad_height, dar_n, dar_d, GST_VIDEO_INFO_PAR_N (&vagg_pad->info), GST_VIDEO_INFO_PAR_D (&vagg_pad->info), GST_VIDEO_INFO_PAR_N (&vagg->info), GST_VIDEO_INFO_PAR_D (&vagg->info)); if (pad_height % dar_n == 0) { - pad_height = gst_util_uint64_scale_int (pad_width, dar_n, dar_d); + pad_width = gst_util_uint64_scale_int (pad_height, dar_n, dar_d); } else if (pad_width % dar_d == 0) { - pad_width = gst_util_uint64_scale_int (pad_height, dar_d, dar_n); + pad_height = gst_util_uint64_scale_int (pad_width, dar_d, dar_n); } else { - pad_height = gst_util_uint64_scale_int (pad_width, dar_n, dar_d); + pad_width = gst_util_uint64_scale_int (pad_height, dar_n, dar_d); } if (width) From b27379051a47bb4233f56d7c79af60ef1d88eac9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Thu, 21 May 2015 16:19:08 +0300 Subject: [PATCH 179/381] compositor: Fix double assignment --- gst/compositor/compositor.c | 5 ----- 1 file changed, 5 deletions(-) diff --git a/gst/compositor/compositor.c b/gst/compositor/compositor.c index dd1ba7071d..4967cdc501 100644 --- a/gst/compositor/compositor.c +++ b/gst/compositor/compositor.c @@ -252,11 +252,6 @@ _mixer_pad_get_output_size (GstCompositor * comp, *width = pad_width; if (height) *height = pad_height; - - if (width) - *width = pad_width; - if (height) - *height = pad_height; } static gboolean From a3b806de4e7b4f08e2f757449d036395632cd0b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Thu, 21 May 2015 16:24:48 +0300 Subject: [PATCH 180/381] compositor/glvideomixer: Don't calculate PAR/DAR with unset GstVideoInfos Otherwise we divide by zero. --- ext/gl/gstglvideomixer.c | 9 +++++++++ gst/compositor/compositor.c | 9 +++++++++ 2 files changed, 18 insertions(+) diff --git a/ext/gl/gstglvideomixer.c b/ext/gl/gstglvideomixer.c index 148c702610..6ae693df18 100644 --- a/ext/gl/gstglvideomixer.c +++ b/ext/gl/gstglvideomixer.c @@ -630,6 +630,15 @@ _mixer_pad_get_output_size (GstGLVideoMixer * mix, gint pad_width, pad_height; guint dar_n, dar_d; + /* FIXME: Anything better we can do here? */ + if (!vagg_pad->info.finfo + || vagg_pad->info.finfo->format == GST_VIDEO_FORMAT_UNKNOWN) { + GST_DEBUG_OBJECT (mix_pad, "Have no caps yet"); + *width = 0; + *height = 0; + return; + } + pad_width = mix_pad->width <= 0 ? GST_VIDEO_INFO_WIDTH (&vagg_pad->info) : mix_pad->width; diff --git a/gst/compositor/compositor.c b/gst/compositor/compositor.c index 4967cdc501..dfeb379fb1 100644 --- a/gst/compositor/compositor.c +++ b/gst/compositor/compositor.c @@ -223,6 +223,15 @@ _mixer_pad_get_output_size (GstCompositor * comp, gint pad_width, pad_height; guint dar_n, dar_d; + /* FIXME: Anything better we can do here? */ + if (!vagg_pad->info.finfo + || vagg_pad->info.finfo->format == GST_VIDEO_FORMAT_UNKNOWN) { + GST_DEBUG_OBJECT (comp_pad, "Have no caps yet"); + *width = 0; + *height = 0; + return; + } + pad_width = comp_pad->width <= 0 ? GST_VIDEO_INFO_WIDTH (&vagg_pad->info) : comp_pad->width; From 70bae08cdd5d75cdb5f2f6633fdd11a7a75612be Mon Sep 17 00:00:00 2001 From: Jan Schmidt Date: Mon, 4 May 2015 18:17:21 +1000 Subject: [PATCH 181/381] videoaggregator: Catch errors, and allow sub-class to return NULL from get_output_buffer() A return value of GST_FLOW_OK with a NULL buffer from get_output_buffer() means the sub-class doesn't want to produce an output buffer, so skip it. If gst_videoaggregator_do_aggregate() generates an error, make sure to propagate it - don't just ignore and discard the error by over-writing it with the gst_pad_push() result. --- gst-libs/gst/video/gstvideoaggregator.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index d31690fcf1..2f720f382e 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -1198,6 +1198,10 @@ gst_videoaggregator_do_aggregate (GstVideoAggregator * vagg, gst_flow_get_name (ret)); return ret; } + if (*outbuf == NULL) { + /* sub-class doesn't want to generate output right now */ + return GST_FLOW_OK; + } GST_BUFFER_TIMESTAMP (*outbuf) = output_start_time; GST_BUFFER_DURATION (*outbuf) = output_end_time - output_start_time; @@ -1369,6 +1373,8 @@ gst_videoaggregator_aggregate (GstAggregator * agg, gboolean timeout) if (jitter <= 0) { ret = gst_videoaggregator_do_aggregate (vagg, output_start_time, output_end_time, &outbuf); + if (ret != GST_FLOW_OK) + goto done; vagg->priv->qos_processed++; } else { GstMessage *msg; @@ -1405,6 +1411,8 @@ gst_videoaggregator_aggregate (GstAggregator * agg, gboolean timeout) goto done_unlocked; done: + if (outbuf) + gst_buffer_unref (outbuf); GST_VIDEO_AGGREGATOR_UNLOCK (vagg); done_unlocked: From 03aec73bd4dc7ae206a78012336c5d4dbb77d78e Mon Sep 17 00:00:00 2001 From: Jan Schmidt Date: Sun, 10 May 2015 18:55:16 +1000 Subject: [PATCH 182/381] videoaggregator: Add class property to disable caps scaling Add preserve_update_caps_result boolean on the class to allow sub-classes to disable videoaggregator removing sizes and framerate from the update_caps() return result. --- gst-libs/gst/video/gstvideoaggregator.c | 16 ++++++++++------ gst-libs/gst/video/gstvideoaggregator.h | 8 +++++++- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index 2f720f382e..6c154bdfc6 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -28,7 +28,7 @@ * biggest incoming video stream and the framerate of the fastest incoming one. * * VideoAggregator will do colorspace conversion. - * + * * Zorder for each input stream can be configured on the * #GstVideoAggregatorPad. * @@ -674,6 +674,7 @@ gst_videoaggregator_update_src_caps (GstVideoAggregator * vagg) GstCaps *caps, *peercaps, *info_caps; GstStructure *s; GstVideoInfo info; + int i; if (GST_VIDEO_INFO_FPS_N (&vagg->info) != best_fps_n || GST_VIDEO_INFO_FPS_D (&vagg->info) != best_fps_d) { @@ -711,11 +712,8 @@ gst_videoaggregator_update_src_caps (GstVideoAggregator * vagg) caps = info_caps; } - peercaps = gst_pad_peer_query_caps (agg->srcpad, NULL); - if (peercaps) { - GstCaps *tmp; - int i; - + /* If the sub-class allows it, allow size/framerate changes */ + if (!vagg_klass->preserve_update_caps_result) { s = gst_caps_get_structure (caps, 0); gst_structure_get (s, "width", G_TYPE_INT, &best_width, "height", G_TYPE_INT, &best_height, NULL); @@ -726,6 +724,12 @@ gst_videoaggregator_update_src_caps (GstVideoAggregator * vagg) "height", GST_TYPE_INT_RANGE, 1, G_MAXINT, "framerate", GST_TYPE_FRACTION_RANGE, 0, 1, G_MAXINT, 1, NULL); } + } + + peercaps = gst_pad_peer_query_caps (agg->srcpad, caps); + if (peercaps) { + GstCaps *tmp; + tmp = gst_caps_intersect (caps, peercaps); GST_DEBUG_OBJECT (vagg, "intersecting %" GST_PTR_FORMAT diff --git a/gst-libs/gst/video/gstvideoaggregator.h b/gst-libs/gst/video/gstvideoaggregator.h index 2be7eb6dc0..b3c5c53e78 100644 --- a/gst-libs/gst/video/gstvideoaggregator.h +++ b/gst-libs/gst/video/gstvideoaggregator.h @@ -17,7 +17,7 @@ * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, * Boston, MA 02110-1301, USA. */ - + #ifndef __GST_VIDEO_AGGREGATOR_H__ #define __GST_VIDEO_AGGREGATOR_H__ @@ -86,6 +86,9 @@ struct _GstVideoAggregator * Notifies subclasses what caps format has been negotiated * @find_best_format: Optional. * Lets subclasses decide of the best common format to use. + * @preserve_update_caps_result: Sub-classes should set this to true if the return result + * of the update_caps() method should not be further modified + * by GstVideoAggregator by removing fields. **/ struct _GstVideoAggregatorClass { @@ -105,6 +108,9 @@ struct _GstVideoAggregatorClass GstCaps * downstream_caps, GstVideoInfo * best_info, gboolean * at_least_one_alpha); + + gboolean preserve_update_caps_result; + /* < private > */ gpointer _gst_reserved[GST_PADDING_LARGE]; }; From e9f055f95bdfa1744dee88deab47ae24dff5fb41 Mon Sep 17 00:00:00 2001 From: Thiago Santos Date: Fri, 5 Jun 2015 09:35:39 -0300 Subject: [PATCH 183/381] Fix a common typo: retreive -> retrieve Seems to have been copy pasted around a few places --- ext/gl/gstglmixerbin.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/gl/gstglmixerbin.c b/ext/gl/gstglmixerbin.c index ae8abef95a..d18c2d31e4 100644 --- a/ext/gl/gstglmixerbin.c +++ b/ext/gl/gstglmixerbin.c @@ -483,7 +483,7 @@ gst_gl_mixer_bin_change_state (GstElement * element, GstStateChange transition) gst_gl_mixer_bin_signals[SIGNAL_CREATE_ELEMENT], 0, &self->mixer); if (!self->mixer) { - GST_ERROR_OBJECT (element, "Failed to retreive element"); + GST_ERROR_OBJECT (element, "Failed to retrieve element"); GST_OBJECT_UNLOCK (element); return GST_STATE_CHANGE_FAILURE; } From 5a494a9389bd7541f8513b870b2766c8a95ed387 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Sun, 7 Jun 2015 10:55:35 +0200 Subject: [PATCH 184/381] Release 1.5.1 --- gst/compositor/compositororc-dist.c | 296 +++++++++++++++++----------- 1 file changed, 184 insertions(+), 112 deletions(-) diff --git a/gst/compositor/compositororc-dist.c b/gst/compositor/compositororc-dist.c index db71b932e9..41744820b6 100644 --- a/gst/compositor/compositororc-dist.c +++ b/gst/compositor/compositororc-dist.c @@ -403,7 +403,7 @@ compositor_orc_blend_u8 (guint8 * ORC_RESTRICT d1, int d1_stride, /* 6: mullw */ var41.i = (var40.i * var36.i) & 0xffff; /* 7: shlw */ - var42.i = var38.i << 8; + var42.i = ((orc_uint16) var38.i) << 8; /* 8: addw */ var43.i = var42.i + var41.i; /* 9: shruw */ @@ -460,7 +460,7 @@ _backup_compositor_orc_blend_u8 (OrcExecutor * ORC_RESTRICT ex) /* 6: mullw */ var41.i = (var40.i * var36.i) & 0xffff; /* 7: shlw */ - var42.i = var38.i << 8; + var42.i = ((orc_uint16) var38.i) << 8; /* 8: addw */ var43.i = var42.i + var41.i; /* 9: shruw */ @@ -606,8 +606,9 @@ compositor_orc_blend_argb (guint8 * ORC_RESTRICT d1, int d1_stride, var43 = var42.i; /* 3: splatbl */ var44.i = - ((var43 & 0xff) << 24) | ((var43 & 0xff) << 16) | ((var43 & 0xff) << - 8) | (var43 & 0xff); + ((((orc_uint32) var43) & 0xff) << 24) | ((((orc_uint32) var43) & 0xff) + << 16) | ((((orc_uint32) var43) & 0xff) << 8) | (((orc_uint32) var43) + & 0xff); /* 4: convubw */ var45.x4[0] = (orc_uint8) var44.x4[0]; var45.x4[1] = (orc_uint8) var44.x4[1]; @@ -618,11 +619,19 @@ compositor_orc_blend_argb (guint8 * ORC_RESTRICT d1, int d1_stride, var46.x4[1] = (var45.x4[1] * var39.x4[1]) & 0xffff; var46.x4[2] = (var45.x4[2] * var39.x4[2]) & 0xffff; var46.x4[3] = (var45.x4[3] * var39.x4[3]) & 0xffff; - /* 7: shruw */ - var47.x4[0] = ((orc_uint16) var46.x4[0]) >> 8; - var47.x4[1] = ((orc_uint16) var46.x4[1]) >> 8; - var47.x4[2] = ((orc_uint16) var46.x4[2]) >> 8; - var47.x4[3] = ((orc_uint16) var46.x4[3]) >> 8; + /* 7: div255w */ + var47.x4[0] = + ((orc_uint16) (((orc_uint16) (var46.x4[0] + 128)) + + (((orc_uint16) (var46.x4[0] + 128)) >> 8))) >> 8; + var47.x4[1] = + ((orc_uint16) (((orc_uint16) (var46.x4[1] + 128)) + + (((orc_uint16) (var46.x4[1] + 128)) >> 8))) >> 8; + var47.x4[2] = + ((orc_uint16) (((orc_uint16) (var46.x4[2] + 128)) + + (((orc_uint16) (var46.x4[2] + 128)) >> 8))) >> 8; + var47.x4[3] = + ((orc_uint16) (((orc_uint16) (var46.x4[3] + 128)) + + (((orc_uint16) (var46.x4[3] + 128)) >> 8))) >> 8; /* 8: convubw */ var48.x4[0] = (orc_uint8) var41.x4[0]; var48.x4[1] = (orc_uint8) var41.x4[1]; @@ -731,8 +740,9 @@ _backup_compositor_orc_blend_argb (OrcExecutor * ORC_RESTRICT ex) var43 = var42.i; /* 3: splatbl */ var44.i = - ((var43 & 0xff) << 24) | ((var43 & 0xff) << 16) | ((var43 & 0xff) << - 8) | (var43 & 0xff); + ((((orc_uint32) var43) & 0xff) << 24) | ((((orc_uint32) var43) & 0xff) + << 16) | ((((orc_uint32) var43) & 0xff) << 8) | (((orc_uint32) var43) + & 0xff); /* 4: convubw */ var45.x4[0] = (orc_uint8) var44.x4[0]; var45.x4[1] = (orc_uint8) var44.x4[1]; @@ -743,11 +753,19 @@ _backup_compositor_orc_blend_argb (OrcExecutor * ORC_RESTRICT ex) var46.x4[1] = (var45.x4[1] * var39.x4[1]) & 0xffff; var46.x4[2] = (var45.x4[2] * var39.x4[2]) & 0xffff; var46.x4[3] = (var45.x4[3] * var39.x4[3]) & 0xffff; - /* 7: shruw */ - var47.x4[0] = ((orc_uint16) var46.x4[0]) >> 8; - var47.x4[1] = ((orc_uint16) var46.x4[1]) >> 8; - var47.x4[2] = ((orc_uint16) var46.x4[2]) >> 8; - var47.x4[3] = ((orc_uint16) var46.x4[3]) >> 8; + /* 7: div255w */ + var47.x4[0] = + ((orc_uint16) (((orc_uint16) (var46.x4[0] + 128)) + + (((orc_uint16) (var46.x4[0] + 128)) >> 8))) >> 8; + var47.x4[1] = + ((orc_uint16) (((orc_uint16) (var46.x4[1] + 128)) + + (((orc_uint16) (var46.x4[1] + 128)) >> 8))) >> 8; + var47.x4[2] = + ((orc_uint16) (((orc_uint16) (var46.x4[2] + 128)) + + (((orc_uint16) (var46.x4[2] + 128)) >> 8))) >> 8; + var47.x4[3] = + ((orc_uint16) (((orc_uint16) (var46.x4[3] + 128)) + + (((orc_uint16) (var46.x4[3] + 128)) >> 8))) >> 8; /* 8: convubw */ var48.x4[0] = (orc_uint8) var41.x4[0]; var48.x4[1] = (orc_uint8) var41.x4[1]; @@ -820,13 +838,13 @@ compositor_orc_blend_argb (guint8 * ORC_RESTRICT d1, int d1_stride, static const orc_uint8 bc[] = { 1, 7, 9, 25, 99, 111, 109, 112, 111, 115, 105, 116, 111, 114, 95, 111, 114, 99, 95, 98, 108, 101, 110, 100, 95, 97, 114, 103, 98, 11, 4, 4, - 12, 4, 4, 14, 4, 255, 0, 0, 0, 14, 4, 8, 0, 0, 0, 16, - 2, 20, 4, 20, 2, 20, 1, 20, 4, 20, 8, 20, 8, 20, 8, 113, - 32, 4, 163, 33, 32, 157, 34, 33, 152, 35, 34, 21, 2, 150, 38, 35, - 21, 2, 89, 38, 38, 24, 21, 2, 95, 38, 38, 17, 21, 2, 150, 37, - 32, 113, 32, 0, 21, 2, 150, 36, 32, 21, 2, 98, 37, 37, 36, 21, - 2, 89, 37, 37, 38, 21, 2, 80, 37, 37, 21, 2, 70, 36, 36, 37, - 21, 2, 157, 32, 36, 123, 32, 32, 16, 128, 0, 32, 2, 0, + 12, 4, 4, 14, 4, 255, 0, 0, 0, 16, 2, 20, 4, 20, 2, 20, + 1, 20, 4, 20, 8, 20, 8, 20, 8, 113, 32, 4, 163, 33, 32, 157, + 34, 33, 152, 35, 34, 21, 2, 150, 38, 35, 21, 2, 89, 38, 38, 24, + 21, 2, 80, 38, 38, 21, 2, 150, 37, 32, 113, 32, 0, 21, 2, 150, + 36, 32, 21, 2, 98, 37, 37, 36, 21, 2, 89, 37, 37, 38, 21, 2, + 80, 37, 37, 21, 2, 70, 36, 36, 37, 21, 2, 157, 32, 36, 123, 32, + 32, 16, 128, 0, 32, 2, 0, }; p = orc_program_new_from_static_bytecode (bc); orc_program_set_backup_function (p, _backup_compositor_orc_blend_argb); @@ -838,7 +856,6 @@ compositor_orc_blend_argb (guint8 * ORC_RESTRICT d1, int d1_stride, orc_program_add_destination (p, 4, "d1"); orc_program_add_source (p, 4, "s1"); orc_program_add_constant (p, 4, 0x000000ff, "c1"); - orc_program_add_constant (p, 4, 0x00000008, "c2"); orc_program_add_parameter (p, 2, "p1"); orc_program_add_temporary (p, 4, "t1"); orc_program_add_temporary (p, 2, "t2"); @@ -860,7 +877,7 @@ compositor_orc_blend_argb (guint8 * ORC_RESTRICT d1, int d1_stride, ORC_VAR_D1); orc_program_append_2 (p, "mullw", 2, ORC_VAR_T7, ORC_VAR_T7, ORC_VAR_P1, ORC_VAR_D1); - orc_program_append_2 (p, "shruw", 2, ORC_VAR_T7, ORC_VAR_T7, ORC_VAR_C2, + orc_program_append_2 (p, "div255w", 2, ORC_VAR_T7, ORC_VAR_T7, ORC_VAR_D1, ORC_VAR_D1); orc_program_append_2 (p, "convubw", 2, ORC_VAR_T6, ORC_VAR_T1, ORC_VAR_D1, ORC_VAR_D1); @@ -965,8 +982,9 @@ compositor_orc_blend_bgra (guint8 * ORC_RESTRICT d1, int d1_stride, var45 = var44.i; /* 4: splatbl */ var46.i = - ((var45 & 0xff) << 24) | ((var45 & 0xff) << 16) | ((var45 & 0xff) << - 8) | (var45 & 0xff); + ((((orc_uint32) var45) & 0xff) << 24) | ((((orc_uint32) var45) & 0xff) + << 16) | ((((orc_uint32) var45) & 0xff) << 8) | (((orc_uint32) var45) + & 0xff); /* 5: convubw */ var47.x4[0] = (orc_uint8) var46.x4[0]; var47.x4[1] = (orc_uint8) var46.x4[1]; @@ -977,11 +995,19 @@ compositor_orc_blend_bgra (guint8 * ORC_RESTRICT d1, int d1_stride, var48.x4[1] = (var47.x4[1] * var40.x4[1]) & 0xffff; var48.x4[2] = (var47.x4[2] * var40.x4[2]) & 0xffff; var48.x4[3] = (var47.x4[3] * var40.x4[3]) & 0xffff; - /* 8: shruw */ - var49.x4[0] = ((orc_uint16) var48.x4[0]) >> 8; - var49.x4[1] = ((orc_uint16) var48.x4[1]) >> 8; - var49.x4[2] = ((orc_uint16) var48.x4[2]) >> 8; - var49.x4[3] = ((orc_uint16) var48.x4[3]) >> 8; + /* 8: div255w */ + var49.x4[0] = + ((orc_uint16) (((orc_uint16) (var48.x4[0] + 128)) + + (((orc_uint16) (var48.x4[0] + 128)) >> 8))) >> 8; + var49.x4[1] = + ((orc_uint16) (((orc_uint16) (var48.x4[1] + 128)) + + (((orc_uint16) (var48.x4[1] + 128)) >> 8))) >> 8; + var49.x4[2] = + ((orc_uint16) (((orc_uint16) (var48.x4[2] + 128)) + + (((orc_uint16) (var48.x4[2] + 128)) >> 8))) >> 8; + var49.x4[3] = + ((orc_uint16) (((orc_uint16) (var48.x4[3] + 128)) + + (((orc_uint16) (var48.x4[3] + 128)) >> 8))) >> 8; /* 9: convubw */ var50.x4[0] = (orc_uint8) var42.x4[0]; var50.x4[1] = (orc_uint8) var42.x4[1]; @@ -1093,8 +1119,9 @@ _backup_compositor_orc_blend_bgra (OrcExecutor * ORC_RESTRICT ex) var45 = var44.i; /* 4: splatbl */ var46.i = - ((var45 & 0xff) << 24) | ((var45 & 0xff) << 16) | ((var45 & 0xff) << - 8) | (var45 & 0xff); + ((((orc_uint32) var45) & 0xff) << 24) | ((((orc_uint32) var45) & 0xff) + << 16) | ((((orc_uint32) var45) & 0xff) << 8) | (((orc_uint32) var45) + & 0xff); /* 5: convubw */ var47.x4[0] = (orc_uint8) var46.x4[0]; var47.x4[1] = (orc_uint8) var46.x4[1]; @@ -1105,11 +1132,19 @@ _backup_compositor_orc_blend_bgra (OrcExecutor * ORC_RESTRICT ex) var48.x4[1] = (var47.x4[1] * var40.x4[1]) & 0xffff; var48.x4[2] = (var47.x4[2] * var40.x4[2]) & 0xffff; var48.x4[3] = (var47.x4[3] * var40.x4[3]) & 0xffff; - /* 8: shruw */ - var49.x4[0] = ((orc_uint16) var48.x4[0]) >> 8; - var49.x4[1] = ((orc_uint16) var48.x4[1]) >> 8; - var49.x4[2] = ((orc_uint16) var48.x4[2]) >> 8; - var49.x4[3] = ((orc_uint16) var48.x4[3]) >> 8; + /* 8: div255w */ + var49.x4[0] = + ((orc_uint16) (((orc_uint16) (var48.x4[0] + 128)) + + (((orc_uint16) (var48.x4[0] + 128)) >> 8))) >> 8; + var49.x4[1] = + ((orc_uint16) (((orc_uint16) (var48.x4[1] + 128)) + + (((orc_uint16) (var48.x4[1] + 128)) >> 8))) >> 8; + var49.x4[2] = + ((orc_uint16) (((orc_uint16) (var48.x4[2] + 128)) + + (((orc_uint16) (var48.x4[2] + 128)) >> 8))) >> 8; + var49.x4[3] = + ((orc_uint16) (((orc_uint16) (var48.x4[3] + 128)) + + (((orc_uint16) (var48.x4[3] + 128)) >> 8))) >> 8; /* 9: convubw */ var50.x4[0] = (orc_uint8) var42.x4[0]; var50.x4[1] = (orc_uint8) var42.x4[1]; @@ -1182,14 +1217,14 @@ compositor_orc_blend_bgra (guint8 * ORC_RESTRICT d1, int d1_stride, static const orc_uint8 bc[] = { 1, 7, 9, 25, 99, 111, 109, 112, 111, 115, 105, 116, 111, 114, 95, 111, 114, 99, 95, 98, 108, 101, 110, 100, 95, 98, 103, 114, 97, 11, 4, 4, - 12, 4, 4, 14, 4, 0, 0, 0, 255, 14, 4, 24, 0, 0, 0, 14, - 4, 8, 0, 0, 0, 16, 2, 20, 4, 20, 4, 20, 2, 20, 1, 20, - 4, 20, 8, 20, 8, 20, 8, 113, 32, 4, 126, 33, 32, 17, 163, 34, - 33, 157, 35, 34, 152, 36, 35, 21, 2, 150, 39, 36, 21, 2, 89, 39, - 39, 24, 21, 2, 95, 39, 39, 18, 21, 2, 150, 38, 32, 113, 32, 0, - 21, 2, 150, 37, 32, 21, 2, 98, 38, 38, 37, 21, 2, 89, 38, 38, - 39, 21, 2, 80, 38, 38, 21, 2, 70, 37, 37, 38, 21, 2, 157, 32, - 37, 123, 32, 32, 16, 128, 0, 32, 2, 0, + 12, 4, 4, 14, 4, 0, 0, 0, 255, 14, 4, 24, 0, 0, 0, 16, + 2, 20, 4, 20, 4, 20, 2, 20, 1, 20, 4, 20, 8, 20, 8, 20, + 8, 113, 32, 4, 126, 33, 32, 17, 163, 34, 33, 157, 35, 34, 152, 36, + 35, 21, 2, 150, 39, 36, 21, 2, 89, 39, 39, 24, 21, 2, 80, 39, + 39, 21, 2, 150, 38, 32, 113, 32, 0, 21, 2, 150, 37, 32, 21, 2, + 98, 38, 38, 37, 21, 2, 89, 38, 38, 39, 21, 2, 80, 38, 38, 21, + 2, 70, 37, 37, 38, 21, 2, 157, 32, 37, 123, 32, 32, 16, 128, 0, + 32, 2, 0, }; p = orc_program_new_from_static_bytecode (bc); orc_program_set_backup_function (p, _backup_compositor_orc_blend_bgra); @@ -1202,7 +1237,6 @@ compositor_orc_blend_bgra (guint8 * ORC_RESTRICT d1, int d1_stride, orc_program_add_source (p, 4, "s1"); orc_program_add_constant (p, 4, 0xff000000, "c1"); orc_program_add_constant (p, 4, 0x00000018, "c2"); - orc_program_add_constant (p, 4, 0x00000008, "c3"); orc_program_add_parameter (p, 2, "p1"); orc_program_add_temporary (p, 4, "t1"); orc_program_add_temporary (p, 4, "t2"); @@ -1227,7 +1261,7 @@ compositor_orc_blend_bgra (guint8 * ORC_RESTRICT d1, int d1_stride, ORC_VAR_D1); orc_program_append_2 (p, "mullw", 2, ORC_VAR_T8, ORC_VAR_T8, ORC_VAR_P1, ORC_VAR_D1); - orc_program_append_2 (p, "shruw", 2, ORC_VAR_T8, ORC_VAR_T8, ORC_VAR_C3, + orc_program_append_2 (p, "div255w", 2, ORC_VAR_T8, ORC_VAR_T8, ORC_VAR_D1, ORC_VAR_D1); orc_program_append_2 (p, "convubw", 2, ORC_VAR_T7, ORC_VAR_T1, ORC_VAR_D1, ORC_VAR_D1); @@ -1351,8 +1385,9 @@ compositor_orc_overlay_argb (guint8 * ORC_RESTRICT d1, int d1_stride, var46 = var45.i; /* 3: splatbl */ var47.i = - ((var46 & 0xff) << 24) | ((var46 & 0xff) << 16) | ((var46 & 0xff) << - 8) | (var46 & 0xff); + ((((orc_uint32) var46) & 0xff) << 24) | ((((orc_uint32) var46) & 0xff) + << 16) | ((((orc_uint32) var46) & 0xff) << 8) | (((orc_uint32) var46) + & 0xff); /* 4: convubw */ var48.x4[0] = (orc_uint8) var47.x4[0]; var48.x4[1] = (orc_uint8) var47.x4[1]; @@ -1363,11 +1398,19 @@ compositor_orc_overlay_argb (guint8 * ORC_RESTRICT d1, int d1_stride, var49.x4[1] = (var48.x4[1] * var41.x4[1]) & 0xffff; var49.x4[2] = (var48.x4[2] * var41.x4[2]) & 0xffff; var49.x4[3] = (var48.x4[3] * var41.x4[3]) & 0xffff; - /* 7: shruw */ - var50.x4[0] = ((orc_uint16) var49.x4[0]) >> 8; - var50.x4[1] = ((orc_uint16) var49.x4[1]) >> 8; - var50.x4[2] = ((orc_uint16) var49.x4[2]) >> 8; - var50.x4[3] = ((orc_uint16) var49.x4[3]) >> 8; + /* 7: div255w */ + var50.x4[0] = + ((orc_uint16) (((orc_uint16) (var49.x4[0] + 128)) + + (((orc_uint16) (var49.x4[0] + 128)) >> 8))) >> 8; + var50.x4[1] = + ((orc_uint16) (((orc_uint16) (var49.x4[1] + 128)) + + (((orc_uint16) (var49.x4[1] + 128)) >> 8))) >> 8; + var50.x4[2] = + ((orc_uint16) (((orc_uint16) (var49.x4[2] + 128)) + + (((orc_uint16) (var49.x4[2] + 128)) >> 8))) >> 8; + var50.x4[3] = + ((orc_uint16) (((orc_uint16) (var49.x4[3] + 128)) + + (((orc_uint16) (var49.x4[3] + 128)) >> 8))) >> 8; /* 8: convubw */ var51.x4[0] = (orc_uint8) var44.x4[0]; var51.x4[1] = (orc_uint8) var44.x4[1]; @@ -1396,8 +1439,9 @@ compositor_orc_overlay_argb (guint8 * ORC_RESTRICT d1, int d1_stride, var58 = var57.i; /* 16: splatbl */ var59.i = - ((var58 & 0xff) << 24) | ((var58 & 0xff) << 16) | ((var58 & 0xff) << - 8) | (var58 & 0xff); + ((((orc_uint32) var58) & 0xff) << 24) | ((((orc_uint32) var58) & 0xff) + << 16) | ((((orc_uint32) var58) & 0xff) << 8) | (((orc_uint32) var58) + & 0xff); /* 17: convubw */ var60.x4[0] = (orc_uint8) var59.x4[0]; var60.x4[1] = (orc_uint8) var59.x4[1]; @@ -1557,8 +1601,9 @@ _backup_compositor_orc_overlay_argb (OrcExecutor * ORC_RESTRICT ex) var46 = var45.i; /* 3: splatbl */ var47.i = - ((var46 & 0xff) << 24) | ((var46 & 0xff) << 16) | ((var46 & 0xff) << - 8) | (var46 & 0xff); + ((((orc_uint32) var46) & 0xff) << 24) | ((((orc_uint32) var46) & 0xff) + << 16) | ((((orc_uint32) var46) & 0xff) << 8) | (((orc_uint32) var46) + & 0xff); /* 4: convubw */ var48.x4[0] = (orc_uint8) var47.x4[0]; var48.x4[1] = (orc_uint8) var47.x4[1]; @@ -1569,11 +1614,19 @@ _backup_compositor_orc_overlay_argb (OrcExecutor * ORC_RESTRICT ex) var49.x4[1] = (var48.x4[1] * var41.x4[1]) & 0xffff; var49.x4[2] = (var48.x4[2] * var41.x4[2]) & 0xffff; var49.x4[3] = (var48.x4[3] * var41.x4[3]) & 0xffff; - /* 7: shruw */ - var50.x4[0] = ((orc_uint16) var49.x4[0]) >> 8; - var50.x4[1] = ((orc_uint16) var49.x4[1]) >> 8; - var50.x4[2] = ((orc_uint16) var49.x4[2]) >> 8; - var50.x4[3] = ((orc_uint16) var49.x4[3]) >> 8; + /* 7: div255w */ + var50.x4[0] = + ((orc_uint16) (((orc_uint16) (var49.x4[0] + 128)) + + (((orc_uint16) (var49.x4[0] + 128)) >> 8))) >> 8; + var50.x4[1] = + ((orc_uint16) (((orc_uint16) (var49.x4[1] + 128)) + + (((orc_uint16) (var49.x4[1] + 128)) >> 8))) >> 8; + var50.x4[2] = + ((orc_uint16) (((orc_uint16) (var49.x4[2] + 128)) + + (((orc_uint16) (var49.x4[2] + 128)) >> 8))) >> 8; + var50.x4[3] = + ((orc_uint16) (((orc_uint16) (var49.x4[3] + 128)) + + (((orc_uint16) (var49.x4[3] + 128)) >> 8))) >> 8; /* 8: convubw */ var51.x4[0] = (orc_uint8) var44.x4[0]; var51.x4[1] = (orc_uint8) var44.x4[1]; @@ -1602,8 +1655,9 @@ _backup_compositor_orc_overlay_argb (OrcExecutor * ORC_RESTRICT ex) var58 = var57.i; /* 16: splatbl */ var59.i = - ((var58 & 0xff) << 24) | ((var58 & 0xff) << 16) | ((var58 & 0xff) << - 8) | (var58 & 0xff); + ((((orc_uint32) var58) & 0xff) << 24) | ((((orc_uint32) var58) & 0xff) + << 16) | ((((orc_uint32) var58) & 0xff) << 8) | (((orc_uint32) var58) + & 0xff); /* 17: convubw */ var60.x4[0] = (orc_uint8) var59.x4[0]; var60.x4[1] = (orc_uint8) var59.x4[1]; @@ -1706,17 +1760,17 @@ compositor_orc_overlay_argb (guint8 * ORC_RESTRICT d1, int d1_stride, 1, 7, 9, 27, 99, 111, 109, 112, 111, 115, 105, 116, 111, 114, 95, 111, 114, 99, 95, 111, 118, 101, 114, 108, 97, 121, 95, 97, 114, 103, 98, 11, 4, 4, 12, 4, 4, 14, 4, 255, 255, 255, 255, 14, 4, 255, 0, 0, - 0, 14, 4, 0, 255, 255, 255, 14, 4, 8, 0, 0, 0, 16, 2, 20, - 4, 20, 2, 20, 1, 20, 8, 20, 8, 20, 8, 20, 4, 20, 8, 20, - 8, 113, 32, 4, 163, 33, 32, 157, 34, 33, 152, 38, 34, 21, 2, 150, - 35, 38, 21, 2, 89, 35, 35, 24, 21, 2, 95, 35, 35, 19, 21, 2, - 150, 40, 32, 21, 2, 89, 40, 40, 35, 115, 38, 16, 21, 2, 150, 36, - 38, 21, 2, 98, 36, 36, 35, 113, 32, 0, 163, 33, 32, 157, 34, 33, - 152, 38, 34, 21, 2, 150, 37, 38, 21, 2, 89, 37, 37, 36, 21, 2, - 80, 37, 37, 21, 2, 150, 39, 32, 21, 2, 89, 39, 39, 37, 21, 2, - 70, 39, 39, 40, 21, 2, 70, 37, 37, 35, 21, 2, 81, 39, 39, 37, - 21, 2, 157, 32, 39, 106, 32, 32, 18, 21, 2, 157, 38, 37, 106, 38, - 38, 17, 123, 32, 32, 38, 128, 0, 32, 2, 0, + 0, 14, 4, 0, 255, 255, 255, 16, 2, 20, 4, 20, 2, 20, 1, 20, + 8, 20, 8, 20, 8, 20, 4, 20, 8, 20, 8, 113, 32, 4, 163, 33, + 32, 157, 34, 33, 152, 38, 34, 21, 2, 150, 35, 38, 21, 2, 89, 35, + 35, 24, 21, 2, 80, 35, 35, 21, 2, 150, 40, 32, 21, 2, 89, 40, + 40, 35, 115, 38, 16, 21, 2, 150, 36, 38, 21, 2, 98, 36, 36, 35, + 113, 32, 0, 163, 33, 32, 157, 34, 33, 152, 38, 34, 21, 2, 150, 37, + 38, 21, 2, 89, 37, 37, 36, 21, 2, 80, 37, 37, 21, 2, 150, 39, + 32, 21, 2, 89, 39, 39, 37, 21, 2, 70, 39, 39, 40, 21, 2, 70, + 37, 37, 35, 21, 2, 81, 39, 39, 37, 21, 2, 157, 32, 39, 106, 32, + 32, 18, 21, 2, 157, 38, 37, 106, 38, 38, 17, 123, 32, 32, 38, 128, + 0, 32, 2, 0, }; p = orc_program_new_from_static_bytecode (bc); orc_program_set_backup_function (p, _backup_compositor_orc_overlay_argb); @@ -1730,7 +1784,6 @@ compositor_orc_overlay_argb (guint8 * ORC_RESTRICT d1, int d1_stride, orc_program_add_constant (p, 4, 0xffffffff, "c1"); orc_program_add_constant (p, 4, 0x000000ff, "c2"); orc_program_add_constant (p, 4, 0xffffff00, "c3"); - orc_program_add_constant (p, 4, 0x00000008, "c4"); orc_program_add_parameter (p, 2, "p1"); orc_program_add_temporary (p, 4, "t1"); orc_program_add_temporary (p, 2, "t2"); @@ -1754,7 +1807,7 @@ compositor_orc_overlay_argb (guint8 * ORC_RESTRICT d1, int d1_stride, ORC_VAR_D1); orc_program_append_2 (p, "mullw", 2, ORC_VAR_T4, ORC_VAR_T4, ORC_VAR_P1, ORC_VAR_D1); - orc_program_append_2 (p, "shruw", 2, ORC_VAR_T4, ORC_VAR_T4, ORC_VAR_C4, + orc_program_append_2 (p, "div255w", 2, ORC_VAR_T4, ORC_VAR_T4, ORC_VAR_D1, ORC_VAR_D1); orc_program_append_2 (p, "convubw", 2, ORC_VAR_T9, ORC_VAR_T1, ORC_VAR_D1, ORC_VAR_D1); @@ -1908,8 +1961,9 @@ compositor_orc_overlay_bgra (guint8 * ORC_RESTRICT d1, int d1_stride, var48 = var47.i; /* 4: splatbl */ var49.i = - ((var48 & 0xff) << 24) | ((var48 & 0xff) << 16) | ((var48 & 0xff) << - 8) | (var48 & 0xff); + ((((orc_uint32) var48) & 0xff) << 24) | ((((orc_uint32) var48) & 0xff) + << 16) | ((((orc_uint32) var48) & 0xff) << 8) | (((orc_uint32) var48) + & 0xff); /* 5: convubw */ var50.x4[0] = (orc_uint8) var49.x4[0]; var50.x4[1] = (orc_uint8) var49.x4[1]; @@ -1920,11 +1974,19 @@ compositor_orc_overlay_bgra (guint8 * ORC_RESTRICT d1, int d1_stride, var51.x4[1] = (var50.x4[1] * var42.x4[1]) & 0xffff; var51.x4[2] = (var50.x4[2] * var42.x4[2]) & 0xffff; var51.x4[3] = (var50.x4[3] * var42.x4[3]) & 0xffff; - /* 8: shruw */ - var52.x4[0] = ((orc_uint16) var51.x4[0]) >> 8; - var52.x4[1] = ((orc_uint16) var51.x4[1]) >> 8; - var52.x4[2] = ((orc_uint16) var51.x4[2]) >> 8; - var52.x4[3] = ((orc_uint16) var51.x4[3]) >> 8; + /* 8: div255w */ + var52.x4[0] = + ((orc_uint16) (((orc_uint16) (var51.x4[0] + 128)) + + (((orc_uint16) (var51.x4[0] + 128)) >> 8))) >> 8; + var52.x4[1] = + ((orc_uint16) (((orc_uint16) (var51.x4[1] + 128)) + + (((orc_uint16) (var51.x4[1] + 128)) >> 8))) >> 8; + var52.x4[2] = + ((orc_uint16) (((orc_uint16) (var51.x4[2] + 128)) + + (((orc_uint16) (var51.x4[2] + 128)) >> 8))) >> 8; + var52.x4[3] = + ((orc_uint16) (((orc_uint16) (var51.x4[3] + 128)) + + (((orc_uint16) (var51.x4[3] + 128)) >> 8))) >> 8; /* 9: convubw */ var53.x4[0] = (orc_uint8) var45.x4[0]; var53.x4[1] = (orc_uint8) var45.x4[1]; @@ -1955,8 +2017,9 @@ compositor_orc_overlay_bgra (guint8 * ORC_RESTRICT d1, int d1_stride, var61 = var60.i; /* 18: splatbl */ var62.i = - ((var61 & 0xff) << 24) | ((var61 & 0xff) << 16) | ((var61 & 0xff) << - 8) | (var61 & 0xff); + ((((orc_uint32) var61) & 0xff) << 24) | ((((orc_uint32) var61) & 0xff) + << 16) | ((((orc_uint32) var61) & 0xff) << 8) | (((orc_uint32) var61) + & 0xff); /* 19: convubw */ var63.x4[0] = (orc_uint8) var62.x4[0]; var63.x4[1] = (orc_uint8) var62.x4[1]; @@ -2120,8 +2183,9 @@ _backup_compositor_orc_overlay_bgra (OrcExecutor * ORC_RESTRICT ex) var48 = var47.i; /* 4: splatbl */ var49.i = - ((var48 & 0xff) << 24) | ((var48 & 0xff) << 16) | ((var48 & 0xff) << - 8) | (var48 & 0xff); + ((((orc_uint32) var48) & 0xff) << 24) | ((((orc_uint32) var48) & 0xff) + << 16) | ((((orc_uint32) var48) & 0xff) << 8) | (((orc_uint32) var48) + & 0xff); /* 5: convubw */ var50.x4[0] = (orc_uint8) var49.x4[0]; var50.x4[1] = (orc_uint8) var49.x4[1]; @@ -2132,11 +2196,19 @@ _backup_compositor_orc_overlay_bgra (OrcExecutor * ORC_RESTRICT ex) var51.x4[1] = (var50.x4[1] * var42.x4[1]) & 0xffff; var51.x4[2] = (var50.x4[2] * var42.x4[2]) & 0xffff; var51.x4[3] = (var50.x4[3] * var42.x4[3]) & 0xffff; - /* 8: shruw */ - var52.x4[0] = ((orc_uint16) var51.x4[0]) >> 8; - var52.x4[1] = ((orc_uint16) var51.x4[1]) >> 8; - var52.x4[2] = ((orc_uint16) var51.x4[2]) >> 8; - var52.x4[3] = ((orc_uint16) var51.x4[3]) >> 8; + /* 8: div255w */ + var52.x4[0] = + ((orc_uint16) (((orc_uint16) (var51.x4[0] + 128)) + + (((orc_uint16) (var51.x4[0] + 128)) >> 8))) >> 8; + var52.x4[1] = + ((orc_uint16) (((orc_uint16) (var51.x4[1] + 128)) + + (((orc_uint16) (var51.x4[1] + 128)) >> 8))) >> 8; + var52.x4[2] = + ((orc_uint16) (((orc_uint16) (var51.x4[2] + 128)) + + (((orc_uint16) (var51.x4[2] + 128)) >> 8))) >> 8; + var52.x4[3] = + ((orc_uint16) (((orc_uint16) (var51.x4[3] + 128)) + + (((orc_uint16) (var51.x4[3] + 128)) >> 8))) >> 8; /* 9: convubw */ var53.x4[0] = (orc_uint8) var45.x4[0]; var53.x4[1] = (orc_uint8) var45.x4[1]; @@ -2167,8 +2239,9 @@ _backup_compositor_orc_overlay_bgra (OrcExecutor * ORC_RESTRICT ex) var61 = var60.i; /* 18: splatbl */ var62.i = - ((var61 & 0xff) << 24) | ((var61 & 0xff) << 16) | ((var61 & 0xff) << - 8) | (var61 & 0xff); + ((((orc_uint32) var61) & 0xff) << 24) | ((((orc_uint32) var61) & 0xff) + << 16) | ((((orc_uint32) var61) & 0xff) << 8) | (((orc_uint32) var61) + & 0xff); /* 19: convubw */ var63.x4[0] = (orc_uint8) var62.x4[0]; var63.x4[1] = (orc_uint8) var62.x4[1]; @@ -2271,18 +2344,18 @@ compositor_orc_overlay_bgra (guint8 * ORC_RESTRICT d1, int d1_stride, 1, 7, 9, 27, 99, 111, 109, 112, 111, 115, 105, 116, 111, 114, 95, 111, 114, 99, 95, 111, 118, 101, 114, 108, 97, 121, 95, 98, 103, 114, 97, 11, 4, 4, 12, 4, 4, 14, 4, 255, 255, 255, 255, 14, 4, 0, 0, 0, - 255, 14, 4, 255, 255, 255, 0, 14, 4, 24, 0, 0, 0, 14, 4, 8, - 0, 0, 0, 16, 2, 20, 4, 20, 4, 20, 2, 20, 1, 20, 8, 20, - 8, 20, 8, 20, 4, 20, 8, 20, 8, 113, 32, 4, 126, 33, 32, 19, - 163, 34, 33, 157, 35, 34, 152, 39, 35, 21, 2, 150, 36, 39, 21, 2, - 89, 36, 36, 24, 21, 2, 95, 36, 36, 20, 21, 2, 150, 41, 32, 21, - 2, 89, 41, 41, 36, 115, 39, 16, 21, 2, 150, 37, 39, 21, 2, 98, - 37, 37, 36, 113, 32, 0, 126, 33, 32, 19, 163, 34, 33, 157, 35, 34, - 152, 39, 35, 21, 2, 150, 38, 39, 21, 2, 89, 38, 38, 37, 21, 2, - 80, 38, 38, 21, 2, 150, 40, 32, 21, 2, 89, 40, 40, 38, 21, 2, - 70, 40, 40, 41, 21, 2, 70, 38, 38, 36, 21, 2, 81, 40, 40, 38, - 21, 2, 157, 32, 40, 106, 32, 32, 18, 21, 2, 157, 39, 38, 106, 39, - 39, 17, 123, 32, 32, 39, 128, 0, 32, 2, 0, + 255, 14, 4, 255, 255, 255, 0, 14, 4, 24, 0, 0, 0, 16, 2, 20, + 4, 20, 4, 20, 2, 20, 1, 20, 8, 20, 8, 20, 8, 20, 4, 20, + 8, 20, 8, 113, 32, 4, 126, 33, 32, 19, 163, 34, 33, 157, 35, 34, + 152, 39, 35, 21, 2, 150, 36, 39, 21, 2, 89, 36, 36, 24, 21, 2, + 80, 36, 36, 21, 2, 150, 41, 32, 21, 2, 89, 41, 41, 36, 115, 39, + 16, 21, 2, 150, 37, 39, 21, 2, 98, 37, 37, 36, 113, 32, 0, 126, + 33, 32, 19, 163, 34, 33, 157, 35, 34, 152, 39, 35, 21, 2, 150, 38, + 39, 21, 2, 89, 38, 38, 37, 21, 2, 80, 38, 38, 21, 2, 150, 40, + 32, 21, 2, 89, 40, 40, 38, 21, 2, 70, 40, 40, 41, 21, 2, 70, + 38, 38, 36, 21, 2, 81, 40, 40, 38, 21, 2, 157, 32, 40, 106, 32, + 32, 18, 21, 2, 157, 39, 38, 106, 39, 39, 17, 123, 32, 32, 39, 128, + 0, 32, 2, 0, }; p = orc_program_new_from_static_bytecode (bc); orc_program_set_backup_function (p, _backup_compositor_orc_overlay_bgra); @@ -2297,7 +2370,6 @@ compositor_orc_overlay_bgra (guint8 * ORC_RESTRICT d1, int d1_stride, orc_program_add_constant (p, 4, 0xff000000, "c2"); orc_program_add_constant (p, 4, 0x00ffffff, "c3"); orc_program_add_constant (p, 4, 0x00000018, "c4"); - orc_program_add_constant (p, 4, 0x00000008, "c5"); orc_program_add_parameter (p, 2, "p1"); orc_program_add_temporary (p, 4, "t1"); orc_program_add_temporary (p, 4, "t2"); @@ -2324,7 +2396,7 @@ compositor_orc_overlay_bgra (guint8 * ORC_RESTRICT d1, int d1_stride, ORC_VAR_D1); orc_program_append_2 (p, "mullw", 2, ORC_VAR_T5, ORC_VAR_T5, ORC_VAR_P1, ORC_VAR_D1); - orc_program_append_2 (p, "shruw", 2, ORC_VAR_T5, ORC_VAR_T5, ORC_VAR_C5, + orc_program_append_2 (p, "div255w", 2, ORC_VAR_T5, ORC_VAR_T5, ORC_VAR_D1, ORC_VAR_D1); orc_program_append_2 (p, "convubw", 2, ORC_VAR_T10, ORC_VAR_T1, ORC_VAR_D1, ORC_VAR_D1); From 34ffdbeee0b2eef25dccba66be8e1f1a545f9ed0 Mon Sep 17 00:00:00 2001 From: Thiago Santos Date: Wed, 6 May 2015 13:08:11 -0300 Subject: [PATCH 185/381] videoaggregator: use macro to access aggregator src pad Makes code a bit more readable --- gst-libs/gst/video/gstvideoaggregator.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index 6c154bdfc6..110d022dca 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -816,7 +816,7 @@ gst_videoaggregator_pad_sink_setcaps (GstPad * pad, GstObject * parent, } vaggpad->info = info; - gst_pad_mark_reconfigure (vagg->aggregator.srcpad); + gst_pad_mark_reconfigure (GST_AGGREGATOR_SRC_PAD (vagg)); ret = TRUE; GST_VIDEO_AGGREGATOR_UNLOCK (vagg); @@ -1795,7 +1795,7 @@ gst_videoaggregator_release_pad (GstElement * element, GstPad * pad) GST_ELEMENT_CLASS (gst_videoaggregator_parent_class)->release_pad (GST_ELEMENT (vagg), pad); - gst_pad_mark_reconfigure (vagg->aggregator.srcpad); + gst_pad_mark_reconfigure (GST_AGGREGATOR_SRC_PAD (vagg)); GST_VIDEO_AGGREGATOR_UNLOCK (vagg); return; From 958c59f35f6663988e68ef5c2a71e5c2faf2a4a9 Mon Sep 17 00:00:00 2001 From: Thiago Santos Date: Wed, 6 May 2015 13:59:51 -0300 Subject: [PATCH 186/381] videoaggregator: fixing types in aggregate function Correctly use boolean and GstFlowReturn types in the function. --- gst-libs/gst/video/gstvideoaggregator.c | 29 ++++++++++++------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index 110d022dca..b86ee21874 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -1283,17 +1283,18 @@ gst_videoaggregator_get_next_time (GstAggregator * agg) static GstFlowReturn gst_videoaggregator_aggregate (GstAggregator * agg, gboolean timeout) { - GstFlowReturn ret; GstVideoAggregator *vagg = GST_VIDEO_AGGREGATOR (agg); GstClockTime output_start_time, output_end_time; GstBuffer *outbuf = NULL; - gint res; + GstFlowReturn flow_ret; gint64 jitter; GST_VIDEO_AGGREGATOR_LOCK (vagg); if (GST_VIDEO_INFO_FORMAT (&vagg->info) == GST_VIDEO_FORMAT_UNKNOWN || gst_pad_check_reconfigure (agg->srcpad)) { + gboolean ret; + ret = gst_videoaggregator_update_converters (vagg); if (ret) ret = gst_videoaggregator_update_src_caps (vagg); @@ -1351,33 +1352,31 @@ gst_videoaggregator_aggregate (GstAggregator * agg, gboolean timeout) output_end_time = MIN (output_end_time, agg->segment.stop); if (output_end_time == output_start_time) { - res = GST_FLOW_EOS; + flow_ret = GST_FLOW_EOS; } else { - res = + flow_ret = gst_videoaggregator_fill_queues (vagg, output_start_time, output_end_time); } - if (res == GST_FLOW_NEEDS_DATA && !timeout) { + if (flow_ret == GST_FLOW_NEEDS_DATA && !timeout) { GST_DEBUG_OBJECT (vagg, "Need more data for decisions"); - ret = GST_FLOW_OK; + flow_ret = GST_FLOW_OK; goto done; - } else if (res == GST_FLOW_EOS) { + } else if (flow_ret == GST_FLOW_EOS) { GST_VIDEO_AGGREGATOR_UNLOCK (vagg); GST_DEBUG_OBJECT (vagg, "All sinkpads are EOS -- forwarding"); - ret = GST_FLOW_EOS; goto done_unlocked; - } else if (res == GST_FLOW_ERROR) { + } else if (flow_ret == GST_FLOW_ERROR) { GST_WARNING_OBJECT (vagg, "Error collecting buffers"); - ret = GST_FLOW_ERROR; goto done; } jitter = gst_videoaggregator_do_qos (vagg, output_start_time); if (jitter <= 0) { - ret = gst_videoaggregator_do_aggregate (vagg, output_start_time, + flow_ret = gst_videoaggregator_do_aggregate (vagg, output_start_time, output_end_time, &outbuf); - if (ret != GST_FLOW_OK) + if (flow_ret != GST_FLOW_OK) goto done; vagg->priv->qos_processed++; } else { @@ -1397,7 +1396,7 @@ gst_videoaggregator_aggregate (GstAggregator * agg, gboolean timeout) vagg->priv->qos_processed, vagg->priv->qos_dropped); gst_element_post_message (GST_ELEMENT_CAST (vagg), msg); - ret = GST_FLOW_OK; + flow_ret = GST_FLOW_OK; } agg->segment.position = output_end_time; @@ -1410,7 +1409,7 @@ gst_videoaggregator_aggregate (GstAggregator * agg, gboolean timeout) GST_TIME_FORMAT, GST_TIME_ARGS (GST_BUFFER_TIMESTAMP (outbuf)), GST_TIME_ARGS (GST_BUFFER_DURATION (outbuf))); - ret = gst_aggregator_finish_buffer (agg, outbuf); + flow_ret = gst_aggregator_finish_buffer (agg, outbuf); } goto done_unlocked; @@ -1420,7 +1419,7 @@ done: GST_VIDEO_AGGREGATOR_UNLOCK (vagg); done_unlocked: - return ret; + return flow_ret; } /* FIXME, the duration query should reflect how long you will produce From dd144953322f025240e890299b99f10186c5e79c Mon Sep 17 00:00:00 2001 From: Thiago Santos Date: Wed, 6 May 2015 14:19:36 -0300 Subject: [PATCH 187/381] videoaggregator: refactor caps reconfigure to its own function Makes the aggregation code shorter and easier to read --- gst-libs/gst/video/gstvideoaggregator.c | 42 +++++++++++++++++-------- 1 file changed, 29 insertions(+), 13 deletions(-) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index b86ee21874..25995062e2 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -1281,18 +1281,13 @@ gst_videoaggregator_get_next_time (GstAggregator * agg) } static GstFlowReturn -gst_videoaggregator_aggregate (GstAggregator * agg, gboolean timeout) +gst_videoaggregator_check_reconfigure (GstVideoAggregator * vagg, + gboolean timeout) { - GstVideoAggregator *vagg = GST_VIDEO_AGGREGATOR (agg); - GstClockTime output_start_time, output_end_time; - GstBuffer *outbuf = NULL; - GstFlowReturn flow_ret; - gint64 jitter; - - GST_VIDEO_AGGREGATOR_LOCK (vagg); + GstAggregator *agg = (GstAggregator *) vagg; if (GST_VIDEO_INFO_FORMAT (&vagg->info) == GST_VIDEO_FORMAT_UNKNOWN - || gst_pad_check_reconfigure (agg->srcpad)) { + || gst_pad_check_reconfigure (GST_AGGREGATOR_SRC_PAD (vagg))) { gboolean ret; ret = gst_videoaggregator_update_converters (vagg); @@ -1300,7 +1295,7 @@ gst_videoaggregator_aggregate (GstAggregator * agg, gboolean timeout) ret = gst_videoaggregator_update_src_caps (vagg); if (!ret) { - if (timeout && gst_pad_needs_reconfigure (agg->srcpad)) { + if (timeout && gst_pad_needs_reconfigure (GST_AGGREGATOR_SRC_PAD (vagg))) { guint64 frame_duration; gint fps_d, fps_n; @@ -1328,15 +1323,36 @@ gst_videoaggregator_aggregate (GstAggregator * agg, gboolean timeout) else agg->segment.position = 0; vagg->priv->nframes++; - GST_VIDEO_AGGREGATOR_UNLOCK (vagg); - return GST_FLOW_OK; + return GST_FLOW_NEEDS_DATA; } else { - GST_VIDEO_AGGREGATOR_UNLOCK (vagg); return GST_FLOW_NOT_NEGOTIATED; } } } + return GST_FLOW_OK; +} + +static GstFlowReturn +gst_videoaggregator_aggregate (GstAggregator * agg, gboolean timeout) +{ + GstVideoAggregator *vagg = GST_VIDEO_AGGREGATOR (agg); + GstClockTime output_start_time, output_end_time; + GstBuffer *outbuf = NULL; + GstFlowReturn flow_ret; + gint64 jitter; + + GST_VIDEO_AGGREGATOR_LOCK (vagg); + + flow_ret = gst_videoaggregator_check_reconfigure (vagg, timeout); + if (flow_ret == GST_FLOW_NEEDS_DATA) { + GST_VIDEO_AGGREGATOR_UNLOCK (vagg); + return GST_FLOW_OK; + } else if (flow_ret != GST_FLOW_OK) { + GST_VIDEO_AGGREGATOR_UNLOCK (vagg); + return flow_ret; + } + output_start_time = gst_videoaggregator_get_next_time (agg); if (GST_VIDEO_INFO_FPS_N (&vagg->info) == 0) From 71328df56ff2144a37beecdfbcbc9ac94f0b1843 Mon Sep 17 00:00:00 2001 From: Thiago Santos Date: Wed, 6 May 2015 14:29:01 -0300 Subject: [PATCH 188/381] videoaggregator: simplify aggregate returning Rework special handling with goto/labels to only have one case and otherwise just return normally. --- gst-libs/gst/video/gstvideoaggregator.c | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index 25995062e2..72d346def9 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -1345,12 +1345,10 @@ gst_videoaggregator_aggregate (GstAggregator * agg, gboolean timeout) GST_VIDEO_AGGREGATOR_LOCK (vagg); flow_ret = gst_videoaggregator_check_reconfigure (vagg, timeout); - if (flow_ret == GST_FLOW_NEEDS_DATA) { - GST_VIDEO_AGGREGATOR_UNLOCK (vagg); - return GST_FLOW_OK; - } else if (flow_ret != GST_FLOW_OK) { - GST_VIDEO_AGGREGATOR_UNLOCK (vagg); - return flow_ret; + if (flow_ret != GST_FLOW_OK) { + if (flow_ret == GST_FLOW_NEEDS_DATA) + flow_ret = GST_FLOW_OK; + goto unlock_and_return; } output_start_time = gst_videoaggregator_get_next_time (agg); @@ -1378,14 +1376,13 @@ gst_videoaggregator_aggregate (GstAggregator * agg, gboolean timeout) if (flow_ret == GST_FLOW_NEEDS_DATA && !timeout) { GST_DEBUG_OBJECT (vagg, "Need more data for decisions"); flow_ret = GST_FLOW_OK; - goto done; + goto unlock_and_return; } else if (flow_ret == GST_FLOW_EOS) { - GST_VIDEO_AGGREGATOR_UNLOCK (vagg); GST_DEBUG_OBJECT (vagg, "All sinkpads are EOS -- forwarding"); - goto done_unlocked; + goto unlock_and_return; } else if (flow_ret == GST_FLOW_ERROR) { GST_WARNING_OBJECT (vagg, "Error collecting buffers"); - goto done; + goto unlock_and_return; } jitter = gst_videoaggregator_do_qos (vagg, output_start_time); @@ -1427,14 +1424,13 @@ gst_videoaggregator_aggregate (GstAggregator * agg, gboolean timeout) flow_ret = gst_aggregator_finish_buffer (agg, outbuf); } - goto done_unlocked; + return flow_ret; done: if (outbuf) gst_buffer_unref (outbuf); +unlock_and_return: GST_VIDEO_AGGREGATOR_UNLOCK (vagg); - -done_unlocked: return flow_ret; } From 2014cb4aa79bcde8c5093601c000352320ee0fe1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Thu, 11 Jun 2015 15:22:04 +0200 Subject: [PATCH 189/381] gl: Use gst_object_ref_sink() for gl{filter,mixer,src}bin too --- ext/gl/gstglmixerbin.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ext/gl/gstglmixerbin.c b/ext/gl/gstglmixerbin.c index d18c2d31e4..75816e1493 100644 --- a/ext/gl/gstglmixerbin.c +++ b/ext/gl/gstglmixerbin.c @@ -396,8 +396,10 @@ gst_gl_mixer_bin_set_property (GObject * object, /* FIXME: deal with replacing a mixer */ g_return_if_fail (!self->mixer || (self->mixer == mixer)); self->mixer = mixer; - if (mixer) + if (mixer) { + gst_object_ref_sink (mixer); _connect_mixer_element (self); + } break; } default: From 86705fbbca0a24252045c1881ca19ff4db20b449 Mon Sep 17 00:00:00 2001 From: Nirbheek Chauhan Date: Tue, 10 Feb 2015 00:49:35 +0530 Subject: [PATCH 190/381] videoaggregator: add "ignore-eos" property for input pads When set, it causes videoaggregator to repeatedly aggregate the last buffer on an EOS pad instead of skipping it and outputting silence. This is useful, for instance, while playing back files seamless one after the other, to avoid videoaggregator ever outputting silence (the checkerboard pattern). It is to be noted that if all the pads on videoaggregator have this property set on them, the mixer will never forward EOS downstream for obvious reasons. Hence, at least one pad with 'ignore-eos' set to FALSE must send EOS to the mixer before it will be forwarded downstream. https://bugzilla.gnome.org/show_bug.cgi?id=748946 --- gst-libs/gst/video/gstvideoaggregator.c | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index 72d346def9..7e4341b447 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -54,10 +54,12 @@ static void gst_videoaggregator_reset_qos (GstVideoAggregator * vagg); ****************************************/ #define DEFAULT_PAD_ZORDER 0 +#define DEFAULT_PAD_IGNORE_EOS FALSE enum { PROP_PAD_0, PROP_PAD_ZORDER, + PROP_PAD_IGNORE_EOS, }; @@ -87,6 +89,9 @@ gst_videoaggregator_pad_get_property (GObject * object, guint prop_id, case PROP_PAD_ZORDER: g_value_set_uint (value, pad->zorder); break; + case PROP_PAD_IGNORE_EOS: + g_value_set_boolean (value, pad->ignore_eos); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); break; @@ -116,6 +121,9 @@ gst_videoaggregator_pad_set_property (GObject * object, guint prop_id, (GCompareFunc) pad_zorder_compare); GST_OBJECT_UNLOCK (vagg); break; + case PROP_PAD_IGNORE_EOS: + pad->ignore_eos = g_value_get_boolean (value); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); break; @@ -304,6 +312,11 @@ gst_videoaggregator_pad_class_init (GstVideoAggregatorPadClass * klass) g_param_spec_uint ("zorder", "Z-Order", "Z Order of the picture", 0, 10000, DEFAULT_PAD_ZORDER, G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); + g_object_class_install_property (gobject_class, PROP_PAD_IGNORE_EOS, + g_param_spec_boolean ("ignore-eos", "Ignore EOS", "Aggregate the last " + "frame on pads that are EOS till they are released", + DEFAULT_PAD_IGNORE_EOS, + G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); g_type_class_add_private (klass, sizeof (GstVideoAggregatorPadPrivate)); @@ -322,6 +335,7 @@ gst_videoaggregator_pad_init (GstVideoAggregatorPad * vaggpad) GstVideoAggregatorPadPrivate); vaggpad->zorder = DEFAULT_PAD_ZORDER; + vaggpad->ignore_eos = DEFAULT_PAD_IGNORE_EOS; vaggpad->aggregated_frame = NULL; vaggpad->priv->converted_buffer = NULL; @@ -1112,6 +1126,12 @@ gst_videoaggregator_fill_queues (GstVideoAggregator * vagg, continue; } } else { + if (is_eos && pad->ignore_eos) { + eos = FALSE; + GST_DEBUG_OBJECT (pad, "ignoring EOS and re-using previous buffer"); + continue; + } + if (pad->priv->end_time != -1) { if (pad->priv->end_time <= output_start_time) { pad->priv->start_time = pad->priv->end_time = -1; From 498c92aac4a5e56a13be78ee8b985352c1eecb59 Mon Sep 17 00:00:00 2001 From: Nirbheek Chauhan Date: Sat, 6 Jun 2015 20:40:13 +0530 Subject: [PATCH 191/381] tests: Add test for the 'ignore-eos' compositor sink pad property When the 'ignore-eos' property is set on a pad, compositor will keep resending the last buffer on the pad till the pad is unlinked. We count the buffers received on appsink, and if it's more than the buffers sent by videotestsrc, the test passes. --- tests/check/elements/compositor.c | 98 +++++++++++++++++++++++++++++++ 1 file changed, 98 insertions(+) diff --git a/tests/check/elements/compositor.c b/tests/check/elements/compositor.c index 7ce637eb00..a63ce7fa95 100644 --- a/tests/check/elements/compositor.c +++ b/tests/check/elements/compositor.c @@ -1328,6 +1328,103 @@ GST_START_TEST (test_obscured_skipped) GST_END_TEST; +static gint buffers_sent = 0; + +static void +_pipeline_eos (GstBus * bus, GstMessage * message, GstPipeline * bin) +{ + GST_INFO ("pipeline EOS"); + g_main_loop_quit (main_loop); +} + +static GstFlowReturn +_buffer_recvd (GstElement * appsink, gint * buffers_recvd) +{ + GstSample *sample; + + g_signal_emit_by_name (appsink, "pull-sample", &sample); + ck_assert_msg (sample != NULL, "NULL sample received!"); + + (*buffers_recvd)++; + GST_INFO ("buffer recvd"); + gst_sample_unref (sample); + + if (*buffers_recvd > buffers_sent) + g_main_loop_quit (main_loop); + + return GST_FLOW_OK; +} + +GST_START_TEST (test_ignore_eos) +{ + gboolean res; + gint buffers_recvd; + GstPadLinkReturn link_res; + GstStateChangeReturn state_res; + GstElement *bin, *src, *compositor, *appsink; + GstPad *srcpad, *sinkpad; + GstBus *bus; + + GST_INFO ("preparing test"); + + /* build pipeline */ + bin = gst_pipeline_new ("pipeline"); + bus = gst_element_get_bus (bin); + gst_bus_add_signal_watch_full (bus, G_PRIORITY_HIGH); + + buffers_sent = 5; + src = gst_element_factory_make ("videotestsrc", NULL); + g_object_set (src, "num-buffers", buffers_sent, NULL); + compositor = gst_element_factory_make ("compositor", NULL); + appsink = gst_element_factory_make ("appsink", NULL); + g_object_set (appsink, "emit-signals", TRUE, NULL); + gst_bin_add_many (GST_BIN (bin), src, compositor, appsink, NULL); + + res = gst_element_link (compositor, appsink); + ck_assert_msg (res == TRUE, "Could not link compositor with appsink"); + srcpad = gst_element_get_static_pad (src, "src"); + sinkpad = gst_element_get_request_pad (compositor, "sink_%u"); + /* When "ignore-eos" is set, compositor will keep sending the last buffer even + * after EOS, so we will receive more buffers than we sent. */ + g_object_set (sinkpad, "ignore-eos", TRUE, NULL); + link_res = gst_pad_link (srcpad, sinkpad); + ck_assert_msg (GST_PAD_LINK_SUCCESSFUL (link_res), "videotestsrc -> " + "compositor pad link failed: %i", link_res); + gst_object_unref (sinkpad); + gst_object_unref (srcpad); + + GST_INFO ("pipeline built, connecting signals"); + + buffers_recvd = 0; + state_res = gst_element_set_state (bin, GST_STATE_PLAYING); + ck_assert_msg (state_res != GST_STATE_CHANGE_FAILURE, "Pipeline didn't play"); + + main_loop = g_main_loop_new (NULL, FALSE); + g_signal_connect (bus, "message::error", G_CALLBACK (message_received), bin); + g_signal_connect (bus, "message::warning", G_CALLBACK (message_received), + bin); + g_signal_connect (bus, "message::eos", G_CALLBACK (_pipeline_eos), bin); + g_signal_connect (appsink, "new-sample", G_CALLBACK (_buffer_recvd), + &buffers_recvd); + + GST_INFO ("starting test"); + g_main_loop_run (main_loop); + + state_res = gst_element_set_state (bin, GST_STATE_NULL); + ck_assert_int_ne (state_res, GST_STATE_CHANGE_FAILURE); + + ck_assert_msg (buffers_recvd > buffers_sent, "Did not receive more buffers" + " than were sent"); + + /* cleanup */ + g_main_loop_unref (main_loop); + gst_bus_remove_signal_watch (bus); + gst_object_unref (bus); + gst_object_unref (bin); +} + +GST_END_TEST; + static Suite * compositor_suite (void) { @@ -1348,6 +1445,7 @@ compositor_suite (void) tcase_add_test (tc_chain, test_flush_start_flush_stop); tcase_add_test (tc_chain, test_segment_base_handling); tcase_add_test (tc_chain, test_obscured_skipped); + tcase_add_test (tc_chain, test_ignore_eos); /* Use a longer timeout */ #ifdef HAVE_VALGRIND From acd6b876db09a2da9c97e213ed6d011c43371d4f Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Sat, 13 Jun 2015 18:43:04 +1000 Subject: [PATCH 192/381] glmixerbin: implement proper dynamic pad removal https://bugzilla.gnome.org/show_bug.cgi?id=750881 --- ext/gl/gstglmixerbin.c | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/ext/gl/gstglmixerbin.c b/ext/gl/gstglmixerbin.c index 75816e1493..f56aad6ea7 100644 --- a/ext/gl/gstglmixerbin.c +++ b/ext/gl/gstglmixerbin.c @@ -47,19 +47,17 @@ _free_input_chain (struct input_chain *chain) if (!chain) return; - if (chain->ghost_pad) - gst_object_unref (chain->ghost_pad); chain->ghost_pad = NULL; if (chain->upload) { + gst_element_set_state (chain->upload, GST_STATE_NULL); gst_bin_remove (GST_BIN (chain->self), chain->upload); - gst_object_unref (chain->upload); chain->upload = NULL; } if (chain->in_convert) { + gst_element_set_state (chain->in_convert, GST_STATE_NULL); gst_bin_remove (GST_BIN (chain->self), chain->in_convert); - gst_object_unref (chain->in_convert); chain->in_convert = NULL; } @@ -449,21 +447,25 @@ gst_gl_mixer_bin_release_pad (GstElement * element, GstPad * pad) { GstGLMixerBin *self = GST_GL_MIXER_BIN (element); GList *l = self->priv->input_chains; + gboolean released = FALSE; GST_OBJECT_LOCK (element); while (l) { struct input_chain *chain = l->data; - if (chain->mixer_pad == pad) { - _free_input_chain (chain); + if (GST_PAD (chain->ghost_pad) == pad) { self->priv->input_chains = g_list_remove_link (self->priv->input_chains, l); + GST_OBJECT_UNLOCK (element); + + _free_input_chain (chain); + gst_element_remove_pad (element, pad); + released = TRUE; break; } l = l->next; } - GST_OBJECT_UNLOCK (element); - - gst_element_remove_pad (element, pad); + if (!released) + GST_OBJECT_UNLOCK (element); } static GstStateChangeReturn From 27ebae85f3d2a478e2d6428da180755449494de0 Mon Sep 17 00:00:00 2001 From: Mathieu Duponchelle Date: Sun, 14 Jun 2015 23:13:59 +0200 Subject: [PATCH 193/381] videoaggregator: No need to artificially bound the zorder. It is an unsigned integer so the upper bound is G_MAXUINT. --- gst-libs/gst/video/gstvideoaggregator.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index 7e4341b447..471b854196 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -310,7 +310,7 @@ gst_videoaggregator_pad_class_init (GstVideoAggregatorPadClass * klass) g_object_class_install_property (gobject_class, PROP_PAD_ZORDER, g_param_spec_uint ("zorder", "Z-Order", "Z Order of the picture", - 0, 10000, DEFAULT_PAD_ZORDER, + 0, G_MAXUINT, DEFAULT_PAD_ZORDER, G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); g_object_class_install_property (gobject_class, PROP_PAD_IGNORE_EOS, g_param_spec_boolean ("ignore-eos", "Ignore EOS", "Aggregate the last " From 98c68600974ad2f6641df9fdebf2e3c1c0ab8ea6 Mon Sep 17 00:00:00 2001 From: Mathieu Duponchelle Date: Sun, 14 Jun 2015 23:20:38 +0200 Subject: [PATCH 194/381] compositor: update zorder documentation. It is not bound between 0 and 10000 anymore. --- gst/compositor/compositor.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gst/compositor/compositor.c b/gst/compositor/compositor.c index dfeb379fb1..2e8ab5cc56 100644 --- a/gst/compositor/compositor.c +++ b/gst/compositor/compositor.c @@ -56,8 +56,8 @@ * (#gdouble) * * - * "zorder": The z-order position of the picture in the composition; between - * 0 and 10000. (#guint) + * "zorder": The z-order position of the picture in the composition + * (#guint) * * * From 4f3adad607aa270fb63a93dff5672505b5d2403e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Mon, 15 Jun 2015 14:18:39 +0200 Subject: [PATCH 195/381] videoaggregator: Remember if the last LATENCY query returned live or not and use the in the QoS messages --- gst-libs/gst/video/gstvideoaggregator.c | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index 471b854196..baacdb5dfe 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -425,6 +425,8 @@ struct _GstVideoAggregatorPrivate /* current caps */ GstCaps *current_caps; + + gboolean live; }; G_DEFINE_ABSTRACT_TYPE_WITH_CODE (GstVideoAggregator, gst_videoaggregator, @@ -945,6 +947,7 @@ gst_videoaggregator_reset (GstVideoAggregator * vagg) gst_video_info_init (&vagg->info); vagg->priv->ts_offset = 0; vagg->priv->nframes = 0; + vagg->priv->live = FALSE; agg->segment.position = -1; @@ -1417,9 +1420,8 @@ gst_videoaggregator_aggregate (GstAggregator * agg, gboolean timeout) vagg->priv->qos_dropped++; - /* TODO: live */ msg = - gst_message_new_qos (GST_OBJECT_CAST (vagg), FALSE, + gst_message_new_qos (GST_OBJECT_CAST (vagg), vagg->priv->live, gst_segment_to_running_time (&agg->segment, GST_FORMAT_TIME, output_start_time), gst_segment_to_stream_time (&agg->segment, GST_FORMAT_TIME, output_start_time), output_start_time, @@ -1569,6 +1571,15 @@ gst_videoaggregator_src_query (GstAggregator * agg, GstQuery * query) case GST_QUERY_DURATION: res = gst_videoaggregator_query_duration (vagg, query); break; + case GST_QUERY_LATENCY: + res = + GST_AGGREGATOR_CLASS (gst_videoaggregator_parent_class)->src_query + (agg, query); + + if (res) { + gst_query_parse_latency (query, &vagg->priv->live, NULL, NULL); + } + break; default: res = GST_AGGREGATOR_CLASS (gst_videoaggregator_parent_class)->src_query From 2f3735f3eca1dc0a7adab42c95fc5ae69d634b49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Mon, 15 Jun 2015 14:19:05 +0200 Subject: [PATCH 196/381] videoaggregator: Don't update the ts-offset before updating the actual configured caps --- gst-libs/gst/video/gstvideoaggregator.c | 9 --------- 1 file changed, 9 deletions(-) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index baacdb5dfe..16ef61833e 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -692,14 +692,6 @@ gst_videoaggregator_update_src_caps (GstVideoAggregator * vagg) GstVideoInfo info; int i; - if (GST_VIDEO_INFO_FPS_N (&vagg->info) != best_fps_n || - GST_VIDEO_INFO_FPS_D (&vagg->info) != best_fps_d) { - if (agg->segment.position != -1) { - vagg->priv->ts_offset = agg->segment.position - agg->segment.start; - vagg->priv->nframes = 0; - } - } - /* Initialize the video info with our target format and * the best width and height and framerate. Then copy over * all other fields as we negotiated them before @@ -746,7 +738,6 @@ gst_videoaggregator_update_src_caps (GstVideoAggregator * vagg) if (peercaps) { GstCaps *tmp; - tmp = gst_caps_intersect (caps, peercaps); GST_DEBUG_OBJECT (vagg, "intersecting %" GST_PTR_FORMAT " with peer caps %" GST_PTR_FORMAT " result %" GST_PTR_FORMAT, caps, From 8352e069ff10e3645cfee7b46edd31a3ae6ef07b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Mon, 15 Jun 2015 14:25:43 +0200 Subject: [PATCH 197/381] videoaggregator: Add some more debug output --- gst-libs/gst/video/gstvideoaggregator.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index 16ef61833e..cc1939977d 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -1399,6 +1399,10 @@ gst_videoaggregator_aggregate (GstAggregator * agg, gboolean timeout) goto unlock_and_return; } + GST_DEBUG_OBJECT (vagg, + "Producing buffer for %" GST_TIME_FORMAT " to %" GST_TIME_FORMAT, + GST_TIME_ARGS (output_start_time), GST_TIME_ARGS (output_end_time)); + jitter = gst_videoaggregator_do_qos (vagg, output_start_time); if (jitter <= 0) { flow_ret = gst_videoaggregator_do_aggregate (vagg, output_start_time, From 723eba37609f1a1b4c8f6986e6c28e7df26e5b94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Mon, 15 Jun 2015 14:29:04 +0200 Subject: [PATCH 198/381] videoaggregator: Print some debug output if we change the timestamp offset --- gst-libs/gst/video/gstvideoaggregator.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index cc1939977d..a66ed3a350 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -606,6 +606,10 @@ gst_videoaggregator_src_setcaps (GstVideoAggregator * vagg, GstCaps * caps) if (agg->segment.position != -1) { vagg->priv->ts_offset = agg->segment.position - agg->segment.start; vagg->priv->nframes = 0; + GST_DEBUG_OBJECT (vagg, + "Updating timestamp offset to %" GST_TIME_FORMAT " for segment %" + GST_SEGMENT_FORMAT, GST_TIME_ARGS (vagg->priv->ts_offset), + &agg->segment); } gst_videoaggregator_reset_qos (vagg); } From 92aacf5760cb7f616401120cef6261b93e15d634 Mon Sep 17 00:00:00 2001 From: Jan Schmidt Date: Sat, 30 May 2015 02:29:04 +1000 Subject: [PATCH 199/381] gl: Add glviewconvert, glstereomix and glstereosplit elements Conversion elements for transforming multiview/stereoscopic video https://bugzilla.gnome.org/show_bug.cgi?id=611157 --- ext/gl/gstglstereomix.c | 702 ++++++++++++++++++++++++++++++++++++++++ ext/gl/gstglstereomix.h | 83 +++++ 2 files changed, 785 insertions(+) create mode 100644 ext/gl/gstglstereomix.c create mode 100644 ext/gl/gstglstereomix.h diff --git a/ext/gl/gstglstereomix.c b/ext/gl/gstglstereomix.c new file mode 100644 index 0000000000..1422823f77 --- /dev/null +++ b/ext/gl/gstglstereomix.c @@ -0,0 +1,702 @@ +/* + * Combine video streams to 3D stereo + * + * GStreamer + * Copyright (C) 2009 Julien Isorce + * Copyright (C) 2014 Jan Schmidt + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "gstglstereomix.h" + +#define GST_CAT_DEFAULT gst_gl_stereo_mix_debug +GST_DEBUG_CATEGORY (gst_gl_stereo_mix_debug); + +#define gst_gl_stereo_mix_parent_class parent_class +G_DEFINE_TYPE (GstGLStereoMix, gst_gl_stereo_mix, GST_TYPE_GL_MIXER); + +static GstCaps *_update_caps (GstVideoAggregator * vagg, GstCaps * caps); +static gboolean _negotiated_caps (GstVideoAggregator * videoaggregator, + GstCaps * caps); +gboolean gst_gl_stereo_mix_make_output (GstGLStereoMix * mix); +static gboolean gst_gl_stereo_mix_process_frames (GstGLStereoMix * mixer, + GPtrArray * in_frames); + +#define DEFAULT_DOWNMIX GST_GL_STEREO_DOWNMIX_ANAGLYPH_GREEN_MAGENTA_DUBOIS + +/* GLStereoMix signals and args */ +enum +{ + /* FILL ME */ + LAST_SIGNAL +}; + +enum +{ + PROP_0, + PROP_DOWNMIX_MODE +}; + +static GstStaticPadTemplate src_factory = GST_STATIC_PAD_TEMPLATE ("src", + GST_PAD_SRC, + GST_PAD_ALWAYS, + GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES + (GST_CAPS_FEATURE_MEMORY_GL_MEMORY, + "RGBA") "; " + GST_VIDEO_CAPS_MAKE_WITH_FEATURES + (GST_CAPS_FEATURE_META_GST_VIDEO_GL_TEXTURE_UPLOAD_META, + "RGBA") + "; " GST_VIDEO_CAPS_MAKE (GST_GL_COLOR_CONVERT_FORMATS)) + ); + +static GstStaticPadTemplate sink_factory = GST_STATIC_PAD_TEMPLATE ("sink_%u", + GST_PAD_SINK, + GST_PAD_REQUEST, + GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES + (GST_CAPS_FEATURE_MEMORY_GL_MEMORY, + "RGBA") "; " + GST_VIDEO_CAPS_MAKE_WITH_FEATURES + (GST_CAPS_FEATURE_META_GST_VIDEO_GL_TEXTURE_UPLOAD_META, + "RGBA") + "; " GST_VIDEO_CAPS_MAKE (GST_GL_COLOR_CONVERT_FORMATS)) + ); + +static GstFlowReturn gst_gl_stereo_mix_get_output_buffer (GstVideoAggregator * + videoaggregator, GstBuffer ** outbuf); +static gboolean gst_gl_stereo_mix_stop (GstAggregator * agg); +static gboolean gst_gl_stereo_mix_start (GstAggregator * agg); +static gboolean gst_gl_stereo_mix_src_query (GstAggregator * agg, + GstQuery * query); + +static void +gst_gl_stereo_mix_find_best_format (GstVideoAggregator * vagg, + GstCaps * downstream_caps, GstVideoInfo * best_info, + gboolean * at_least_one_alpha); + +static void gst_gl_stereo_mix_set_property (GObject * object, guint prop_id, + const GValue * value, GParamSpec * pspec); +static void gst_gl_stereo_mix_get_property (GObject * object, guint prop_id, + GValue * value, GParamSpec * pspec); + +static void gst_gl_stereo_mix_finalize (GObject * object); + +static GstFlowReturn +gst_gl_stereo_mix_aggregate_frames (GstVideoAggregator * vagg, + GstBuffer * outbuffer); + +static void +gst_gl_stereo_mix_class_init (GstGLStereoMixClass * klass) +{ + GObjectClass *gobject_class = (GObjectClass *) klass; + GstElementClass *element_class = GST_ELEMENT_CLASS (klass); + GstVideoAggregatorClass *videoaggregator_class = + (GstVideoAggregatorClass *) klass; + GstAggregatorClass *agg_class = (GstAggregatorClass *) klass; + GstGLBaseMixerClass *base_mix_class = (GstGLBaseMixerClass *) klass; + + GST_DEBUG_CATEGORY_INIT (GST_CAT_DEFAULT, "glstereomixer", 0, + "opengl stereoscopic mixer"); + + gobject_class->finalize = GST_DEBUG_FUNCPTR (gst_gl_stereo_mix_finalize); + + gobject_class->get_property = gst_gl_stereo_mix_get_property; + gobject_class->set_property = gst_gl_stereo_mix_set_property; + + gst_element_class_set_metadata (element_class, "OpenGL stereo video combiner", + "Filter/Effect/Video", "OpenGL stereo video combiner", + "Jan Schmidt "); + + g_object_class_install_property (gobject_class, PROP_DOWNMIX_MODE, + g_param_spec_enum ("downmix-mode", "Mode for mono downmixed output", + "Output anaglyph type to generate when downmixing to mono", + GST_TYPE_GL_STEREO_DOWNMIX_MODE_TYPE, DEFAULT_DOWNMIX, + G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + + gst_element_class_add_pad_template (element_class, + gst_static_pad_template_get (&src_factory)); + gst_element_class_add_pad_template (element_class, + gst_static_pad_template_get (&sink_factory)); + + agg_class->stop = gst_gl_stereo_mix_stop; + agg_class->start = gst_gl_stereo_mix_start; + agg_class->src_query = gst_gl_stereo_mix_src_query; + + videoaggregator_class->aggregate_frames = gst_gl_stereo_mix_aggregate_frames; + videoaggregator_class->update_caps = _update_caps; + videoaggregator_class->negotiated_caps = _negotiated_caps; + videoaggregator_class->get_output_buffer = + gst_gl_stereo_mix_get_output_buffer; + videoaggregator_class->find_best_format = gst_gl_stereo_mix_find_best_format; + videoaggregator_class->preserve_update_caps_result = TRUE; + + base_mix_class->supported_gl_api = GST_GL_API_OPENGL | GST_GL_API_OPENGL3; +} + +static void +gst_gl_stereo_mix_init (GstGLStereoMix * mix) +{ +} + +static void +gst_gl_stereo_mix_finalize (GObject * object) +{ + //GstGLStereoMix *mix = GST_GL_STEREO_MIX (object); + + G_OBJECT_CLASS (parent_class)->finalize (object); +} + +static gboolean +gst_gl_stereo_mix_query_caps (GstPad * pad, GstAggregator * agg, + GstQuery * query) +{ + GstCaps *filter, *caps; + + gst_query_parse_caps (query, &filter); + + caps = gst_pad_get_current_caps (agg->srcpad); + if (caps == NULL) { + caps = gst_pad_get_pad_template_caps (agg->srcpad); + } + + if (filter) + caps = gst_caps_intersect_full (filter, caps, GST_CAPS_INTERSECT_FIRST); + + gst_query_set_caps_result (query, caps); + gst_caps_unref (caps); + + return TRUE; +} + +static gboolean +gst_gl_stereo_mix_src_query (GstAggregator * agg, GstQuery * query) +{ + switch (GST_QUERY_TYPE (query)) { + case GST_QUERY_CAPS: + return gst_gl_stereo_mix_query_caps (agg->srcpad, agg, query); + break; + default: + break; + } + + return GST_AGGREGATOR_CLASS (parent_class)->src_query (agg, query); +} + + +static GstFlowReturn +gst_gl_stereo_mix_get_output_buffer (GstVideoAggregator * videoaggregator, + GstBuffer ** outbuf) +{ + GstGLStereoMix *mix = GST_GL_STEREO_MIX (videoaggregator); + GstFlowReturn ret = GST_FLOW_OK; + +#if 0 + + if (!mix->priv->pool_active) { + if (!gst_buffer_pool_set_active (mix->priv->pool, TRUE)) { + GST_ELEMENT_ERROR (mix, RESOURCE, SETTINGS, + ("failed to activate bufferpool"), ("failed to activate bufferpool")); + return GST_FLOW_ERROR; + } + mix->priv->pool_active = TRUE; + } + + return gst_buffer_pool_acquire_buffer (mix->priv->pool, outbuf, NULL); +#endif + + if (!gst_gl_stereo_mix_make_output (mix)) { + gst_buffer_replace (&mix->primary_out, NULL); + gst_buffer_replace (&mix->auxilliary_out, NULL); + GST_ELEMENT_ERROR (mix, RESOURCE, SETTINGS, + ("Failed to generate output"), ("failed to generate output")); + ret = GST_FLOW_ERROR; + } + + if (mix->auxilliary_out) { + *outbuf = mix->auxilliary_out; + mix->auxilliary_out = NULL; + } else { + *outbuf = mix->primary_out; + mix->primary_out = NULL; + } + return ret; +} + +gboolean +gst_gl_stereo_mix_make_output (GstGLStereoMix * mix) +{ + guint i; + GList *walk; + gboolean res = FALSE; + guint array_index = 0; + GstElement *element = GST_ELEMENT (mix); + gboolean missing_buffer = FALSE; + + GST_LOG_OBJECT (mix, "Processing buffers"); + + GST_OBJECT_LOCK (mix); + walk = element->sinkpads; + + i = mix->frames->len; + g_ptr_array_set_size (mix->frames, element->numsinkpads); + for (; i < element->numsinkpads; i++) + mix->frames->pdata[i] = g_slice_new0 (GstGLStereoMixFrameData); + while (walk) { + GstGLMixerPad *pad = GST_GL_MIXER_PAD (walk->data); + GstVideoAggregatorPad *vaggpad = walk->data; + GstGLStereoMixFrameData *frame; + + GST_LOG_OBJECT (mix, "Checking pad %" GST_PTR_FORMAT, vaggpad); + + frame = g_ptr_array_index (mix->frames, array_index); + frame->base.pad = pad; + frame->buf = NULL; + + walk = g_list_next (walk); + + if (vaggpad->buffer != NULL) { + frame->buf = vaggpad->buffer; + + GST_DEBUG_OBJECT (pad, "Got buffer %" GST_PTR_FORMAT, frame->buf); + } else { + GST_LOG_OBJECT (mix, "No buffer on pad %" GST_PTR_FORMAT, vaggpad); + missing_buffer = TRUE; + } + ++array_index; + } + if (missing_buffer) { + /* We're still waiting for a buffer to turn up on at least one input */ + GST_WARNING_OBJECT (mix, "Not generating output - need more input buffers"); + res = TRUE; + goto out; + } + + /* Copy GL memory from each input frame to the output */ + if (!gst_gl_stereo_mix_process_frames (mix, mix->frames)) { + GST_LOG_OBJECT (mix, "Failed to process frames to output"); + goto out; + } + + if (mix->primary_out == NULL) + goto out; + + res = TRUE; + +out: + GST_OBJECT_UNLOCK (mix); + + return res; +} + +static GstFlowReturn +gst_gl_stereo_mix_aggregate_frames (GstVideoAggregator * vagg, + GstBuffer * outbuf) +{ + GstGLStereoMix *mix = GST_GL_STEREO_MIX (vagg); + /* If we're operating in frame-by-frame mode, push + * the primary view now, and let the parent class + * push the remaining auxilliary view */ + if (GST_VIDEO_INFO_MULTIVIEW_MODE (&vagg->info) == + GST_VIDEO_MULTIVIEW_MODE_FRAME_BY_FRAME) { + /* Transfer the timestamps video-agg put on the aux buffer */ + gst_buffer_copy_into (mix->primary_out, outbuf, + GST_BUFFER_COPY_TIMESTAMPS, 0, -1); + gst_aggregator_finish_buffer (GST_AGGREGATOR (vagg), mix->primary_out); + mix->primary_out = NULL; + + /* And actually, we don't want timestamps on the aux buffer */ + GST_BUFFER_TIMESTAMP (outbuf) = GST_CLOCK_TIME_NONE; + GST_BUFFER_DURATION (outbuf) = GST_CLOCK_TIME_NONE; + } + return GST_FLOW_OK; +} + +static void +gst_gl_stereo_mix_get_property (GObject * object, + guint prop_id, GValue * value, GParamSpec * pspec) +{ + GstGLStereoMix *mix = GST_GL_STEREO_MIX (object); + + switch (prop_id) { + case PROP_DOWNMIX_MODE: + g_value_set_enum (value, mix->downmix_mode); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static void +gst_gl_stereo_mix_set_property (GObject * object, + guint prop_id, const GValue * value, GParamSpec * pspec) +{ + GstGLStereoMix *mix = GST_GL_STEREO_MIX (object); + + switch (prop_id) { + case PROP_DOWNMIX_MODE: + mix->downmix_mode = g_value_get_enum (value); + if (mix->viewconvert) + g_object_set_property (G_OBJECT (mix->viewconvert), "downmix-mode", + value); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static void +_free_glmixer_frame_data (GstGLStereoMixFrameData * frame) +{ + if (frame == NULL) + return; + if (frame->buf) + gst_buffer_unref (frame->buf); + g_slice_free1 (sizeof (GstGLStereoMixFrameData), frame); +} + +static gboolean +gst_gl_stereo_mix_start (GstAggregator * agg) +{ + guint i; + GstGLStereoMix *mix = GST_GL_STEREO_MIX (agg); + GstElement *element = GST_ELEMENT (agg); + + if (!GST_AGGREGATOR_CLASS (parent_class)->start (agg)) + return FALSE; + + GST_OBJECT_LOCK (mix); + mix->array_buffers = g_ptr_array_new_full (element->numsinkpads, + (GDestroyNotify) _free_glmixer_frame_data); + mix->frames = g_ptr_array_new_full (element->numsinkpads, NULL); + + g_ptr_array_set_size (mix->array_buffers, element->numsinkpads); + g_ptr_array_set_size (mix->frames, element->numsinkpads); + + for (i = 0; i < element->numsinkpads; i++) + mix->frames->pdata[i] = g_slice_new0 (GstGLStereoMixFrameData); + + mix->viewconvert = gst_gl_view_convert_new (); + g_object_set (G_OBJECT (mix->viewconvert), "downmix-mode", + mix->downmix_mode, NULL); + + GST_OBJECT_UNLOCK (mix); + + return TRUE; +} + +static gboolean +gst_gl_stereo_mix_stop (GstAggregator * agg) +{ + GstGLStereoMix *mix = GST_GL_STEREO_MIX (agg); + + if (!GST_AGGREGATOR_CLASS (parent_class)->stop (agg)) + return FALSE; + + GST_OBJECT_LOCK (agg); + g_ptr_array_free (mix->frames, TRUE); + mix->frames = NULL; + g_ptr_array_free (mix->array_buffers, TRUE); + mix->array_buffers = NULL; + GST_OBJECT_UNLOCK (agg); + + if (mix->viewconvert) { + gst_object_unref (mix->viewconvert); + mix->viewconvert = NULL; + } + + return TRUE; +} + +/* Convert to caps that can be accepted by this element... */ +static GstCaps * +get_converted_caps (GstGLStereoMix * mix, GstCaps * caps) +{ +#if 0 + GstGLContext *context = GST_GL_BASE_MIXER (mix)->context; + GstCaps *result, *tmp; + + GST_LOG_OBJECT (mix, "Converting caps %" GST_PTR_FORMAT, caps); + result = gst_gl_upload_transform_caps (context, GST_PAD_SINK, caps, NULL); + tmp = result; + GST_TRACE_OBJECT (mix, "transfer returned caps %" GST_PTR_FORMAT, tmp); + + result = + gst_gl_color_convert_transform_caps (context, GST_PAD_SINK, tmp, NULL); + gst_caps_unref (tmp); + GST_TRACE_OBJECT (mix, "convert returned caps %" GST_PTR_FORMAT, tmp); + + tmp = result; + result = gst_gl_view_convert_transform_caps (mix->viewconvert, + GST_PAD_SINK, tmp, NULL); + gst_caps_unref (tmp); +#else + GstCaps *result; + + GST_LOG_OBJECT (mix, "Converting caps %" GST_PTR_FORMAT, caps); + result = gst_gl_view_convert_transform_caps (mix->viewconvert, + GST_PAD_SINK, caps, NULL); +#endif + + GST_LOG_OBJECT (mix, "returning caps %" GST_PTR_FORMAT, result); + + return result; +} + +/* Return the possible output caps we decided in find_best_format() */ +static GstCaps * +_update_caps (GstVideoAggregator * vagg, GstCaps * caps) +{ + GstGLStereoMix *mix = GST_GL_STEREO_MIX (vagg); + + return gst_caps_ref (mix->out_caps); +} + +/* Called after videoaggregator fixates our caps */ +static gboolean +_negotiated_caps (GstVideoAggregator * vagg, GstCaps * caps) +{ + GstGLStereoMix *mix = GST_GL_STEREO_MIX (vagg); + + GST_LOG_OBJECT (mix, "Configured output caps %" GST_PTR_FORMAT, caps); + + if (GST_VIDEO_AGGREGATOR_CLASS (parent_class)->negotiated_caps) + if (!GST_VIDEO_AGGREGATOR_CLASS (parent_class)->negotiated_caps (vagg, + caps)) + return FALSE; + + /* Update the glview_convert output */ + if (!gst_video_info_from_caps (&mix->out_info, caps)) + return FALSE; + + /* We can configure the view_converter now */ + gst_gl_view_convert_set_context (mix->viewconvert, + GST_GL_BASE_MIXER (mix)->context); + gst_gl_view_convert_set_format (mix->viewconvert, &mix->mix_info, + &mix->out_info); + + return TRUE; + +} + +static gboolean +gst_gl_stereo_mix_process_frames (GstGLStereoMix * mixer, GPtrArray * frames) +{ + GstVideoAggregator *vagg = GST_VIDEO_AGGREGATOR (mixer); + GstBuffer *converted_buffer, *inbuf; + GstVideoInfo *out_info = &vagg->info; + gint count = 0, n; + gint v, views; + gint valid_views = 0; + + inbuf = gst_buffer_new (); + while (count < frames->len) { + GstGLStereoMixFrameData *frame; + GstMemory *in_mem; + + frame = g_ptr_array_index (frames, count); + GST_LOG_OBJECT (mixer, "Handling frame %d", count); + + if (!frame) { + GST_DEBUG ("skipping texture, null frame"); + count++; + continue; + } + + in_mem = gst_buffer_get_memory (frame->buf, 0); + + GST_LOG_OBJECT (mixer, + "Appending memory %" GST_PTR_FORMAT " to intermediate buffer", in_mem); + /* Appending the memory to a 2nd buffer locks it + * exclusive a 2nd time, which will mark it for + * copy-on-write. The ref will keep the memory + * alive but we add a parent_buffer_meta to also + * prevent the input buffer from returning to any buffer + * pool it might belong to + */ + gst_buffer_append_memory (inbuf, in_mem); + /* Use parent buffer meta to keep input buffer alive */ + gst_buffer_add_parent_buffer_meta (inbuf, frame->buf); + + count++; + valid_views++; + } + + if (mixer->mix_info.views != valid_views) { + GST_WARNING_OBJECT (mixer, "Not enough input views to process"); + return FALSE; + } + + if (GST_VIDEO_INFO_MULTIVIEW_MODE (out_info) == + GST_VIDEO_MULTIVIEW_MODE_SEPARATED) + views = out_info->views; + else + views = 1; + + if (gst_gl_view_convert_submit_input_buffer (mixer->viewconvert, + FALSE, inbuf) != GST_FLOW_OK) + return FALSE; + + /* Clear any existing buffers, just in case */ + gst_buffer_replace (&mixer->primary_out, NULL); + gst_buffer_replace (&mixer->auxilliary_out, NULL); + + if (gst_gl_view_convert_get_output (mixer->viewconvert, + &mixer->primary_out) != GST_FLOW_OK) + return FALSE; + + if (GST_VIDEO_INFO_MULTIVIEW_MODE (out_info) == + GST_VIDEO_MULTIVIEW_MODE_FRAME_BY_FRAME) { + if (gst_gl_view_convert_get_output (mixer->viewconvert, + &mixer->auxilliary_out) != GST_FLOW_OK) + return FALSE; + } + + if (mixer->primary_out == NULL) + return FALSE; + + converted_buffer = mixer->primary_out; + v = 0; + n = gst_buffer_n_memory (converted_buffer); + g_assert (n == GST_VIDEO_INFO_N_PLANES (out_info) * views); + for (v = 0; v < views; v++) { + gst_buffer_add_video_meta_full (converted_buffer, v, + GST_VIDEO_INFO_FORMAT (out_info), + GST_VIDEO_INFO_WIDTH (out_info), + GST_VIDEO_INFO_HEIGHT (out_info), + GST_VIDEO_INFO_N_PLANES (out_info), out_info->offset, out_info->stride); + if (mixer->auxilliary_out) { + gst_buffer_add_video_meta_full (mixer->auxilliary_out, v, + GST_VIDEO_INFO_FORMAT (out_info), + GST_VIDEO_INFO_WIDTH (out_info), + GST_VIDEO_INFO_HEIGHT (out_info), + GST_VIDEO_INFO_N_PLANES (out_info), out_info->offset, + out_info->stride); + } + } + + return TRUE; +} + +/* Iterate the input sink pads, and choose the blend format + * we will generate before output conversion, which is RGBA + * at some suitable size */ +static void +gst_gl_stereo_mix_find_best_format (GstVideoAggregator * vagg, + GstCaps * downstream_caps, GstVideoInfo * best_info, + gboolean * at_least_one_alpha) +{ + GstGLStereoMix *mix = GST_GL_STEREO_MIX (vagg); + GList *l; + gint best_width = -1, best_height = -1; + gdouble best_fps = -1, cur_fps; + gint best_fps_n = 0, best_fps_d = 1; + GstVideoInfo *mix_info; + GstCaps *blend_caps, *tmp_caps; + + /* We'll deal with alpha internally, so just tell aggregator to + * be quiet */ + *at_least_one_alpha = FALSE; + + GST_OBJECT_LOCK (vagg); + + for (l = GST_ELEMENT (vagg)->sinkpads; l; l = l->next) { + GstVideoAggregatorPad *pad = l->data; + GstVideoInfo tmp = pad->info; + gint this_width, this_height; + gint fps_n, fps_d; + + if (!pad->info.finfo) + continue; + + /* This can happen if we release a pad and another pad hasn't been negotiated_caps yet */ + if (GST_VIDEO_INFO_FORMAT (&pad->info) == GST_VIDEO_FORMAT_UNKNOWN) + continue; + + /* Convert to per-view width/height for unpacked forms */ + gst_video_multiview_video_info_change_mode (&tmp, + GST_VIDEO_MULTIVIEW_MODE_SEPARATED, GST_VIDEO_MULTIVIEW_FLAGS_NONE); + + this_width = GST_VIDEO_INFO_WIDTH (&tmp); + this_height = GST_VIDEO_INFO_HEIGHT (&tmp); + fps_n = GST_VIDEO_INFO_FPS_N (&tmp); + fps_d = GST_VIDEO_INFO_FPS_D (&tmp); + + GST_INFO_OBJECT (vagg, "Input pad %" GST_PTR_FORMAT + " w %u h %u", pad, this_width, this_height); + + if (this_width == 0 || this_height == 0) + continue; + + if (best_width < this_width) + best_width = this_width; + if (best_height < this_height) + best_height = this_height; + + if (fps_d == 0) + cur_fps = 0.0; + else + gst_util_fraction_to_double (fps_n, fps_d, &cur_fps); + + if (best_fps < cur_fps) { + best_fps = cur_fps; + best_fps_n = fps_n; + best_fps_d = fps_d; + } + + /* FIXME: Preserve PAR for at least one input when different sized inputs */ + } + GST_OBJECT_UNLOCK (vagg); + + mix_info = &mix->mix_info; + gst_video_info_set_format (mix_info, GST_VIDEO_FORMAT_RGBA, best_width, + best_height); + + GST_VIDEO_INFO_FPS_N (mix_info) = best_fps_n; + GST_VIDEO_INFO_FPS_D (mix_info) = best_fps_d; + + GST_VIDEO_INFO_MULTIVIEW_MODE (mix_info) = GST_VIDEO_MULTIVIEW_MODE_SEPARATED; + GST_VIDEO_INFO_VIEWS (mix_info) = 2; + + /* FIXME: If input is marked as flipped or flopped, preserve those flags */ + GST_VIDEO_INFO_MULTIVIEW_FLAGS (mix_info) = GST_VIDEO_MULTIVIEW_FLAGS_NONE; + + /* Choose our output format based on downstream preferences */ + blend_caps = gst_video_info_to_caps (mix_info); + + gst_caps_set_features (blend_caps, 0, + gst_caps_features_from_string (GST_CAPS_FEATURE_MEMORY_GL_MEMORY)); + + tmp_caps = get_converted_caps (GST_GL_STEREO_MIX (vagg), blend_caps); + gst_caps_unref (blend_caps); + + if (mix->out_caps) + gst_caps_unref (mix->out_caps); + + mix->out_caps = gst_caps_intersect (downstream_caps, tmp_caps); + gst_caps_unref (tmp_caps); + + GST_DEBUG_OBJECT (vagg, "Possible output caps %" GST_PTR_FORMAT, + mix->out_caps); + /* Tell videoaggregator our preferred size. Actual info gets + * overridden during caps nego */ + *best_info = *mix_info; +} diff --git a/ext/gl/gstglstereomix.h b/ext/gl/gstglstereomix.h new file mode 100644 index 0000000000..debe347c49 --- /dev/null +++ b/ext/gl/gstglstereomix.h @@ -0,0 +1,83 @@ +/* + * GStreamer + * Copyright (C) 2009 Julien Isorce + * Copyright (C) 2014 Jan Schmidt + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifndef __GST_GL_STEREO_MIX_H__ +#define __GST_GL_STEREO_MIX_H__ + +#include "gstglmixer.h" + +G_BEGIN_DECLS + +#define GST_TYPE_GL_STEREO_MIX (gst_gl_stereo_mix_get_type()) +#define GST_GL_STEREO_MIX(obj) \ + (G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_GL_STEREO_MIX, GstGLStereoMix)) +#define GST_GL_STEREO_MIX_CLASS(klass) \ + (G_TYPE_CHECK_CLASS_CAST((klass),GST_TYPE_GL_STEREO_MIX, GstGLStereoMixClass)) +#define GST_IS_GL_STEREO_MIX(obj) \ + (G_TYPE_CHECK_INSTANCE_TYPE((obj),GST_TYPE_GL_STEREO_MIX)) +#define GST_IS_GL_STEREO_MIX_CLASS(klass) \ + (G_TYPE_CHECK_CLASS_TYPE((klass),GST_TYPE_GL_STEREO_MIX)) +#define GST_GL_STEREO_MIX_GET_CLASS(obj) \ + (G_TYPE_INSTANCE_GET_CLASS((obj),GST_TYPE_GL_STEREO_MIX,GstGLStereoMixClass)) + +typedef struct _GstGLStereoMix GstGLStereoMix; +typedef struct _GstGLStereoMixClass GstGLStereoMixClass; +typedef struct _GstGLStereoMixFrameData GstGLStereoMixFrameData; + +struct _GstGLStereoMix +{ + GstGLMixer mixer; + + GPtrArray *array_buffers; + GPtrArray *frames; + + GLuint out_tex_id; + GstGLDownload *download; + + GstGLViewConvert *viewconvert; + GstGLStereoDownmix downmix_mode; + + GstCaps *out_caps; + GstVideoInfo out_info; + + GstVideoInfo mix_info; + + GPtrArray *input_frames; + GstBuffer *primary_out; + GstBuffer *auxilliary_out; +}; + +struct _GstGLStereoMixClass +{ + GstGLMixerClass mixer_class; +}; + +struct _GstGLStereoMixFrameData +{ + GstGLMixerFrameData base; + gboolean mapped; + GstBuffer *buf; +}; + +GType gst_gl_stereo_mix_get_type(void); + +G_END_DECLS +#endif /* __GST_GL_STEREO_MIX_H__ */ From fc49a1b508b1759a708b5597133af28ab54b2c10 Mon Sep 17 00:00:00 2001 From: Mathieu Duponchelle Date: Sat, 20 Jun 2015 13:36:27 +0200 Subject: [PATCH 200/381] videoaggregator: simplifies and improves sink_get_caps. The problem here was that after removing the formats and all the things we could convert, we then intersected these caps with the template caps. Hence if a subclass offered permissive sink templates (eg all the possible formats videoconvert handles), but only one output format, then at negotiation time getcaps returned caps with the format restricted to that format, even though we do handle conversion. https://bugzilla.gnome.org/show_bug.cgi?id=751255 --- gst-libs/gst/video/gstvideoaggregator.c | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index a66ed3a350..2943036b30 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -842,7 +842,6 @@ gst_videoaggregator_pad_sink_getcaps (GstPad * pad, GstVideoAggregator * vagg, { GstCaps *srccaps; GstCaps *template_caps; - GstCaps *filtered_caps; GstCaps *returned_caps; GstStructure *s; gboolean had_current_caps = TRUE; @@ -870,14 +869,13 @@ gst_videoaggregator_pad_sink_getcaps (GstPad * pad, GstVideoAggregator * vagg, NULL); } - filtered_caps = srccaps; - if (filter) - filtered_caps = gst_caps_intersect (srccaps, filter); - returned_caps = gst_caps_intersect (filtered_caps, template_caps); + if (filter) { + returned_caps = gst_caps_intersect (srccaps, filter); + gst_caps_unref (srccaps); + } else { + returned_caps = srccaps; + } - gst_caps_unref (srccaps); - if (filter) - gst_caps_unref (filtered_caps); if (had_current_caps) gst_caps_unref (template_caps); From 1c66a33c56a12ba40bab299a376ad3850a5d05c9 Mon Sep 17 00:00:00 2001 From: Nicolas Dufresne Date: Fri, 26 Jun 2015 15:34:35 -0400 Subject: [PATCH 201/381] gl: Don't leak pool if set_config failed --- ext/gl/gstglmixer.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/ext/gl/gstglmixer.c b/ext/gl/gstglmixer.c index 119ab5a359..401e4ed6c9 100644 --- a/ext/gl/gstglmixer.c +++ b/ext/gl/gstglmixer.c @@ -165,13 +165,14 @@ gst_gl_mixer_propose_allocation (GstGLBaseMixer * base_mix, config = gst_buffer_pool_get_config (pool); gst_buffer_pool_config_set_params (config, caps, size, 0, 0); - if (!gst_buffer_pool_set_config (pool, config)) - goto config_failed; - } - if (pool) { + if (!gst_buffer_pool_set_config (pool, config)) { + g_object_unref (pool); + goto config_failed; + } + gst_query_add_allocation_pool (query, pool, size, 1, 0); - gst_object_unref (pool); + g_object_unref (pool); } /* we also support various metadata */ From 50d5d0a161a65ca6bb5d50684044b45e181a0ba7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Olivier=20Cr=C3=AAte?= Date: Fri, 3 Jul 2015 12:17:42 -0400 Subject: [PATCH 202/381] glvideomixer, glmixer: Add description and klass --- ext/gl/gstglmixerbin.c | 4 ++++ ext/gl/gstglvideomixer.c | 5 +++++ 2 files changed, 9 insertions(+) diff --git a/ext/gl/gstglmixerbin.c b/ext/gl/gstglmixerbin.c index f56aad6ea7..c6488b5cd8 100644 --- a/ext/gl/gstglmixerbin.c +++ b/ext/gl/gstglmixerbin.c @@ -177,6 +177,10 @@ gst_gl_mixer_bin_class_init (GstGLMixerBinClass * klass) gst_pad_template_new ("sink_%u", GST_PAD_SINK, GST_PAD_REQUEST, upload_caps)); gst_caps_unref (upload_caps); + + gst_element_class_set_metadata (element_class, "OpenGL video_mixer empty bin", + "Bin/Filter/Effect/Video/Mixer", "OpenGL video_mixer empty bin", + "Matthew Waters "); } static void diff --git a/ext/gl/gstglvideomixer.c b/ext/gl/gstglvideomixer.c index 6ae693df18..3c554802b8 100644 --- a/ext/gl/gstglvideomixer.c +++ b/ext/gl/gstglvideomixer.c @@ -273,6 +273,7 @@ static void gst_gl_video_mixer_bin_class_init (GstGLVideoMixerBinClass * klass) { GstGLMixerBinClass *mixer_class = GST_GL_MIXER_BIN_CLASS (klass); + GstElementClass *element_class = GST_ELEMENT_CLASS (klass); GObjectClass *gobject_class = G_OBJECT_CLASS (klass); mixer_class->create_input_pad = _create_video_mixer_input; @@ -284,6 +285,10 @@ gst_gl_video_mixer_bin_class_init (GstGLVideoMixerBinClass * klass) g_param_spec_enum ("background", "Background", "Background type", GST_GL_TYPE_VIDEO_MIXER_BACKGROUND, DEFAULT_BACKGROUND, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + + gst_element_class_set_metadata (element_class, "OpenGL video_mixer bin", + "Bin/Filter/Effect/Video/Compositor", "OpenGL video_mixer bin", + "Matthew Waters "); } static void From df274deb0b15be11236dcfbb7aa4eb743391904a Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Mon, 6 Jul 2015 18:51:07 +0200 Subject: [PATCH 203/381] videoaggregator: Remove pixel-aspect-ratio field from the caps returned by getcaps Avoiding not negotiated error while negotiating as we anyway force 1/1 as output --- gst-libs/gst/video/gstvideoaggregator.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index 2943036b30..66549d191b 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -866,7 +866,7 @@ gst_videoaggregator_pad_sink_getcaps (GstPad * pad, GstVideoAggregator * vagg, "framerate", GST_TYPE_FRACTION_RANGE, 0, 1, G_MAXINT, 1, NULL); gst_structure_remove_fields (s, "colorimetry", "chroma-site", "format", - NULL); + "pixel-aspect-ratio", NULL); } if (filter) { @@ -1894,7 +1894,7 @@ gst_videoaggregator_pad_sink_acceptcaps (GstPad * pad, "framerate", GST_TYPE_FRACTION_RANGE, 0, 1, G_MAXINT, 1, NULL); gst_structure_remove_fields (s, "colorimetry", "chroma-site", "format", - NULL); + "pixel-aspect-ratio", NULL); } modified_caps = gst_caps_intersect (accepted_caps, template_caps); From 884e68c713bbd7f81885b4f53ac0f741da53a9b8 Mon Sep 17 00:00:00 2001 From: Julien Isorce Date: Fri, 19 Jun 2015 11:57:06 +0100 Subject: [PATCH 204/381] gl: use gst_gl_display_create_context in more elements. glbasefilter, glbasemixer and gltestsrc. https://bugzilla.gnome.org/show_bug.cgi?id=750310 --- ext/gl/gstglbasemixer.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/ext/gl/gstglbasemixer.c b/ext/gl/gstglbasemixer.c index d462c161ad..f8c8d193d5 100644 --- a/ext/gl/gstglbasemixer.c +++ b/ext/gl/gstglbasemixer.c @@ -469,15 +469,16 @@ gst_gl_base_mixer_decide_allocation (GstGLBaseMixer * mix, GstQuery * query) if (!mix->context) { GST_OBJECT_LOCK (mix->display); do { - if (mix->context) + if (mix->context) { gst_object_unref (mix->context); + mix->context = NULL; + } /* just get a GL context. we don't care */ mix->context = gst_gl_display_get_gl_context_for_thread (mix->display, NULL); if (!mix->context) { - mix->context = gst_gl_context_new (mix->display); - if (!gst_gl_context_create (mix->context, mix->priv->other_context, - &error)) { + if (!gst_gl_display_create_context (mix->display, + mix->priv->other_context, &mix->context, &error)) { GST_OBJECT_UNLOCK (mix->display); goto context_error; } From b05002776bb6470a9f001173dc13a33aea55f2be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Olivier=20Cr=C3=AAte?= Date: Thu, 2 Jul 2015 20:10:50 -0400 Subject: [PATCH 205/381] glvideomixer: Add GstControlBinding proxy This is used to proxy GstControlBinding to the pad on the parent object. This avoid having to sync the values in the proxy pad, this is too early if you have a queue between the pad and the actual aggregation operation. https://bugzilla.gnome.org/show_bug.cgi?id=734060 --- ext/gl/gstglvideomixer.c | 212 +++++++++++++++++++++++++++++---------- 1 file changed, 158 insertions(+), 54 deletions(-) diff --git a/ext/gl/gstglvideomixer.c b/ext/gl/gstglvideomixer.c index 3c554802b8..25897d8731 100644 --- a/ext/gl/gstglvideomixer.c +++ b/ext/gl/gstglvideomixer.c @@ -71,6 +71,151 @@ gst_gl_video_mixer_background_get_type (void) return mixer_background_type; } +typedef struct _GstGLMixerControlBindingProxy GstGLMixerControlBindingProxy; +typedef struct _GstGLMixerControlBindingProxyClass + GstGLMixerControlBindingProxyClass; + +struct _GstGLMixerControlBindingProxy +{ + GstControlBinding parent; + + GstObject *ref_object; + const gchar *property_name; +}; + +struct _GstGLMixerControlBindingProxyClass +{ + GstControlBindingClass parent_class; +}; + +GType gst_gl_mixer_control_binding_proxy_get_type (void); +#define GST_TYPE_GL_MIXER_CONTROL_BINDING \ + (gst_gl_mixer_control_binding_proxy_get_type()) + +G_DEFINE_TYPE (GstGLMixerControlBindingProxy, + gst_gl_mixer_control_binding_proxy, GST_TYPE_CONTROL_BINDING); + +static void +gst_gl_mixer_control_binding_proxy_init (GstGLMixerControlBindingProxy * self) +{ +} + +static gboolean +gst_gl_mixer_control_binding_proxy_sync_values (GstControlBinding * binding, + GstObject * object, GstClockTime timestamp, GstClockTime last_sync) +{ + GstGLMixerControlBindingProxy *self = (GstGLMixerControlBindingProxy *) + binding; + GstControlBinding *ref_binding; + gboolean ret = TRUE; + + ref_binding = gst_object_get_control_binding (self->ref_object, + self->property_name); + + if (ref_binding) { + ret = gst_control_binding_sync_values (ref_binding, self->ref_object, + timestamp, last_sync); + gst_object_unref (ref_binding); + } + + return ret; +} + +static GValue * +gst_gl_mixer_control_binding_proxy_get_value (GstControlBinding * binding, + GstClockTime timestamp) +{ + GstGLMixerControlBindingProxy *self = (GstGLMixerControlBindingProxy *) + binding; + GstControlBinding *ref_binding; + GValue *ret = NULL; + + ref_binding = gst_object_get_control_binding (self->ref_object, + self->property_name); + + if (ref_binding) { + ret = gst_control_binding_get_value (ref_binding, timestamp); + gst_object_unref (ref_binding); + } + + return ret; +} + +static gboolean +gst_gl_mixer_control_binding_proxy_get_value_array (GstControlBinding * binding, + GstClockTime timestamp, GstClockTime interval, guint n_values, + gpointer values) +{ + GstGLMixerControlBindingProxy *self = (GstGLMixerControlBindingProxy *) + binding; + GstControlBinding *ref_binding; + gboolean ret = FALSE; + + ref_binding = gst_object_get_control_binding (self->ref_object, + self->property_name); + + if (ref_binding) { + ret = gst_control_binding_get_value_array (ref_binding, timestamp, + interval, n_values, values); + gst_object_unref (ref_binding); + } + + return ret; +} + +static gboolean +gst_gl_mixer_control_binding_proxy_get_g_value_array (GstControlBinding * + binding, GstClockTime timestamp, GstClockTime interval, guint n_values, + GValue * values) +{ + GstGLMixerControlBindingProxy *self = (GstGLMixerControlBindingProxy *) + binding; + GstControlBinding *ref_binding; + gboolean ret = FALSE; + + ref_binding = gst_object_get_control_binding (self->ref_object, + self->property_name); + + if (ref_binding) { + ret = gst_control_binding_get_g_value_array (ref_binding, timestamp, + interval, n_values, values); + gst_object_unref (ref_binding); + } + + return ret; +} + + +static void + gst_gl_mixer_control_binding_proxy_class_init + (GstGLMixerControlBindingProxyClass * klass) +{ + GstControlBindingClass *cb_class = GST_CONTROL_BINDING_CLASS (klass); + + cb_class->sync_values = gst_gl_mixer_control_binding_proxy_sync_values; + cb_class->get_value = gst_gl_mixer_control_binding_proxy_get_value; + cb_class->get_value_array = + gst_gl_mixer_control_binding_proxy_get_value_array; + cb_class->get_g_value_array = + gst_gl_mixer_control_binding_proxy_get_g_value_array; +} + +static GstControlBinding * +gst_gl_mixer_control_binding_proxy_new (GstObject * object, + const gchar * property_name, GstObject * ref_object, + const gchar * ref_property_name) +{ + GstGLMixerControlBindingProxy *self = + g_object_new (GST_TYPE_GL_MIXER_CONTROL_BINDING, "object", object, + "name", property_name, NULL); + + self->ref_object = ref_object; + self->property_name = ref_property_name; + + return (GstControlBinding *) self; +} + + #define DEFAULT_PAD_XPOS 0 #define DEFAULT_PAD_YPOS 0 #define DEFAULT_PAD_WIDTH 0 @@ -94,11 +239,6 @@ static void gst_gl_video_mixer_input_get_property (GObject * object, static void gst_gl_video_mixer_input_set_property (GObject * object, guint prop_id, const GValue * value, GParamSpec * pspec); -static GstFlowReturn gst_gl_video_mixer_input_chain (GstPad * pad, - GstObject * parent, GstBuffer * buffer); -static GstFlowReturn gst_gl_video_mixer_input_event (GstPad * pad, - GstObject * parent, GstEvent * event); - typedef struct _GstGLVideoMixerInput GstGLVideoMixerInput; typedef GstGhostPadClass GstGLVideoMixerInputClass; @@ -118,9 +258,6 @@ G_DEFINE_TYPE (GstGLVideoMixerInput, gst_gl_video_mixer_input, static void gst_gl_video_mixer_input_init (GstGLVideoMixerInput * self) { - GstPad *pad = GST_PAD (self); - - gst_pad_set_event_function (pad, gst_gl_video_mixer_input_event); } static void @@ -177,51 +314,6 @@ gst_gl_video_mixer_input_set_property (GObject * object, guint prop_id, g_object_set_property (G_OBJECT (self->mixer_pad), pspec->name, value); } -static GstFlowReturn -gst_gl_video_mixer_input_chain (GstPad * pad, GstObject * parent, - GstBuffer * buffer) -{ - GstGLVideoMixerInput *self = (GstGLVideoMixerInput *) pad; - GstClockTime timestamp, stream_time; -// gdouble alpha; - - timestamp = GST_BUFFER_TIMESTAMP (buffer); - - stream_time = - gst_segment_to_stream_time (&self->segment, GST_FORMAT_TIME, timestamp); - - gst_object_sync_values (GST_OBJECT (self), stream_time); -#if 0 - /* FIXME: implement no-upload on alpha = 0 */ - g_object_get (self, "alpha", &alpha, NULL); - - if (alpha <= 0.0) { - GST_DEBUG_OBJECT (self, "dropping buffer %" GST_PTR_FORMAT - " due to alpha value %f", buffer, alpha); - gst_buffer_unref (buffer); - return GST_FLOW_OK; - } -#endif - return gst_proxy_pad_chain_default (pad, parent, buffer); -} - -static GstFlowReturn -gst_gl_video_mixer_input_event (GstPad * pad, GstObject * parent, - GstEvent * event) -{ - GstGLVideoMixerInput *self = (GstGLVideoMixerInput *) pad; - - switch (GST_EVENT_TYPE (event)) { - case GST_EVENT_SEGMENT: - gst_event_copy_segment (event, &self->segment); - break; - default: - break; - } - - return gst_pad_event_default (pad, parent, event); -} - static GstGhostPad * _create_video_mixer_input (GstGLMixerBin * self, GstPad * mixer_pad) { @@ -229,13 +321,25 @@ _create_video_mixer_input (GstGLMixerBin * self, GstPad * mixer_pad) g_object_new (gst_gl_video_mixer_input_get_type (), "name", GST_OBJECT_NAME (mixer_pad), "direction", GST_PAD_DIRECTION (mixer_pad), NULL); + GstControlBinding *cb; if (!gst_ghost_pad_construct (GST_GHOST_PAD (input))) { gst_object_unref (input); return NULL; } +#define ADD_PROXY_CONTROL_BINDING(prop) \ + cb = gst_gl_mixer_control_binding_proxy_new (GST_OBJECT (input), \ + G_STRINGIFY (prop), GST_OBJECT (mixer_pad), G_STRINGIFY (prop)); \ + gst_object_add_control_binding (GST_OBJECT (input), cb) - gst_pad_set_chain_function (GST_PAD (input), gst_gl_video_mixer_input_chain); + ADD_PROXY_CONTROL_BINDING (zorder); + ADD_PROXY_CONTROL_BINDING (xpos); + ADD_PROXY_CONTROL_BINDING (ypos); + ADD_PROXY_CONTROL_BINDING (width); + ADD_PROXY_CONTROL_BINDING (height); + ADD_PROXY_CONTROL_BINDING (alpha); + +#undef ADD_PROXY_CONTROL_BINDING input->mixer_pad = mixer_pad; From bd66c6b665d882a65f34cdc0fcaffe8fc865e98c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Mon, 15 Jun 2015 18:30:20 +0200 Subject: [PATCH 206/381] aggregator: Add property to select how to decide on a start time Before aggregator based elements always started at running time 0, now it's possible to select the first input buffer running time or explicitly set a start-time value. https://bugzilla.gnome.org/show_bug.cgi?id=749966 --- gst-libs/gst/video/gstvideoaggregator.c | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index 66549d191b..167c09983e 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -604,12 +604,11 @@ gst_videoaggregator_src_setcaps (GstVideoAggregator * vagg, GstCaps * caps) if (GST_VIDEO_INFO_FPS_N (&vagg->info) != GST_VIDEO_INFO_FPS_N (&info) || GST_VIDEO_INFO_FPS_D (&vagg->info) != GST_VIDEO_INFO_FPS_D (&info)) { if (agg->segment.position != -1) { - vagg->priv->ts_offset = agg->segment.position - agg->segment.start; vagg->priv->nframes = 0; + /* The timestamp offset will be updated based on the + * segment position the next time we aggregate */ GST_DEBUG_OBJECT (vagg, - "Updating timestamp offset to %" GST_TIME_FORMAT " for segment %" - GST_SEGMENT_FORMAT, GST_TIME_ARGS (vagg->priv->ts_offset), - &agg->segment); + "Resetting frame counter because of framerate change"); } gst_videoaggregator_reset_qos (vagg); } @@ -1368,6 +1367,11 @@ gst_videoaggregator_aggregate (GstAggregator * agg, gboolean timeout) } output_start_time = gst_videoaggregator_get_next_time (agg); + if (vagg->priv->nframes == 0) { + vagg->priv->ts_offset = output_start_time; + GST_DEBUG_OBJECT (vagg, "New ts offset %" GST_TIME_FORMAT, + GST_TIME_ARGS (output_start_time)); + } if (GST_VIDEO_INFO_FPS_N (&vagg->info) == 0) output_end_time = -1; @@ -1376,7 +1380,7 @@ gst_videoaggregator_aggregate (GstAggregator * agg, gboolean timeout) vagg->priv->ts_offset + gst_util_uint64_scale_round (vagg->priv->nframes + 1, GST_SECOND * GST_VIDEO_INFO_FPS_D (&vagg->info), - GST_VIDEO_INFO_FPS_N (&vagg->info)) + agg->segment.start; + GST_VIDEO_INFO_FPS_N (&vagg->info)); if (agg->segment.stop != -1) output_end_time = MIN (output_end_time, agg->segment.stop); From dc4c42021759aed7d64f5b7530e9b415660c2b87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Tue, 28 Jul 2015 22:30:09 +0300 Subject: [PATCH 207/381] compositor: Add unit tests for the new aggregator start-time-selection property https://bugzilla.gnome.org/show_bug.cgi?id=749966 --- tests/check/elements/compositor.c | 178 +++++++++++++++++++++++++++++- 1 file changed, 172 insertions(+), 6 deletions(-) diff --git a/tests/check/elements/compositor.c b/tests/check/elements/compositor.c index a63ce7fa95..a949d3828c 100644 --- a/tests/check/elements/compositor.c +++ b/tests/check/elements/compositor.c @@ -1328,8 +1328,6 @@ GST_START_TEST (test_obscured_skipped) GST_END_TEST; -static gint buffers_sent = 0; - static void _pipeline_eos (GstBus * bus, GstMessage * message, GstPipeline * bin) { @@ -1349,7 +1347,7 @@ _buffer_recvd (GstElement * appsink, gint * buffers_recvd) GST_INFO ("buffer recvd"); gst_sample_unref (sample); - if (*buffers_recvd > buffers_sent) + if (*buffers_recvd > 5) g_main_loop_quit (main_loop); return GST_FLOW_OK; @@ -1372,9 +1370,8 @@ GST_START_TEST (test_ignore_eos) bus = gst_element_get_bus (bin); gst_bus_add_signal_watch_full (bus, G_PRIORITY_HIGH); - buffers_sent = 5; src = gst_element_factory_make ("videotestsrc", NULL); - g_object_set (src, "num-buffers", buffers_sent, NULL); + g_object_set (src, "num-buffers", 5, NULL); compositor = gst_element_factory_make ("compositor", NULL); appsink = gst_element_factory_make ("appsink", NULL); g_object_set (appsink, "emit-signals", TRUE, NULL); @@ -1413,7 +1410,7 @@ GST_START_TEST (test_ignore_eos) state_res = gst_element_set_state (bin, GST_STATE_NULL); ck_assert_int_ne (state_res, GST_STATE_CHANGE_FAILURE); - ck_assert_msg (buffers_recvd > buffers_sent, "Did not receive more buffers" + ck_assert_msg (buffers_recvd > 5, "Did not receive more buffers" " than were sent"); /* cleanup */ @@ -1425,6 +1422,169 @@ GST_START_TEST (test_ignore_eos) GST_END_TEST; +typedef struct +{ + gint buffers_sent; + GstClockTime first_pts; + gboolean first; + gboolean drop; +} TestStartTimeSelectionData; + +static GstPadProbeReturn +drop_buffer_cb (GstPad * pad, GstPadProbeInfo * info, gpointer user_data) +{ + TestStartTimeSelectionData *data = user_data; + + if (data->drop) { + data->buffers_sent = data->buffers_sent + 1; + if (data->buffers_sent < 4) + return GST_PAD_PROBE_DROP; + } + + data->first_pts = GST_BUFFER_PTS (info->data); + + return GST_PAD_PROBE_REMOVE; +} + +static GstFlowReturn +first_buffer_received_cb (GstElement * appsink, gpointer user_data) +{ + TestStartTimeSelectionData *data = user_data; + GstSample *sample; + GstBuffer *buffer; + + g_signal_emit_by_name (appsink, "pull-sample", &sample); + ck_assert_msg (sample != NULL, "NULL sample received!"); + + buffer = gst_sample_get_buffer (sample); + if (!data->first) { + fail_unless_equals_uint64 (GST_BUFFER_PTS (buffer), 0); + } else { + fail_unless_equals_uint64 (GST_BUFFER_PTS (buffer), data->first_pts); + } + + gst_sample_unref (sample); + + g_main_loop_quit (main_loop); + + return GST_FLOW_EOS; +} + +static void +run_test_start_time (gboolean first, gboolean drop, gboolean unlinked) +{ + gboolean res; + GstPadLinkReturn link_res; + GstStateChangeReturn state_res; + GstElement *bin, *src, *compositor, *appsink; + GstPad *srcpad, *sinkpad; + GstBus *bus; + TestStartTimeSelectionData data = { 0, GST_CLOCK_TIME_NONE, first, drop }; + + GST_INFO ("preparing test"); + + /* build pipeline */ + bin = gst_pipeline_new ("pipeline"); + bus = gst_element_get_bus (bin); + gst_bus_add_signal_watch_full (bus, G_PRIORITY_HIGH); + + src = gst_element_factory_make ("videotestsrc", NULL); + + srcpad = gst_element_get_static_pad (src, "src"); + gst_pad_add_probe (srcpad, GST_PAD_PROBE_TYPE_BUFFER, drop_buffer_cb, &data, + NULL); + gst_object_unref (srcpad); + + g_object_set (src, "is-live", TRUE, NULL); + compositor = gst_element_factory_make ("compositor", NULL); + g_object_set (compositor, "start-time-selection", (first ? 1 : 0), NULL); + appsink = gst_element_factory_make ("appsink", NULL); + g_object_set (appsink, "emit-signals", TRUE, NULL); + gst_bin_add_many (GST_BIN (bin), src, compositor, appsink, NULL); + + res = gst_element_link (compositor, appsink); + ck_assert_msg (res == TRUE, "Could not link compositor with appsink"); + srcpad = gst_element_get_static_pad (src, "src"); + sinkpad = gst_element_get_request_pad (compositor, "sink_%u"); + link_res = gst_pad_link (srcpad, sinkpad); + ck_assert_msg (GST_PAD_LINK_SUCCESSFUL (link_res), "videotestsrc -> " + "compositor pad link failed: %i", link_res); + gst_object_unref (sinkpad); + gst_object_unref (srcpad); + + if (unlinked) { + sinkpad = gst_element_get_request_pad (compositor, "sink_%u"); + gst_object_unref (sinkpad); + } + + GST_INFO ("pipeline built, connecting signals"); + + state_res = gst_element_set_state (bin, GST_STATE_PLAYING); + ck_assert_msg (state_res != GST_STATE_CHANGE_FAILURE, "Pipeline didn't play"); + + main_loop = g_main_loop_new (NULL, FALSE); + g_signal_connect (bus, "message::error", G_CALLBACK (message_received), bin); + g_signal_connect (bus, "message::warning", G_CALLBACK (message_received), + bin); + g_signal_connect (bus, "message::eos", G_CALLBACK (_pipeline_eos), bin); + g_signal_connect (appsink, "new-sample", + G_CALLBACK (first_buffer_received_cb), &data); + + GST_INFO ("starting test"); + g_main_loop_run (main_loop); + + state_res = gst_element_set_state (bin, GST_STATE_NULL); + ck_assert_int_ne (state_res, GST_STATE_CHANGE_FAILURE); + + /* cleanup */ + g_main_loop_unref (main_loop); + gst_bus_remove_signal_watch (bus); + gst_object_unref (bus); + gst_object_unref (bin); +} + +GST_START_TEST (test_start_time_zero_live_drop_0) +{ + run_test_start_time (FALSE, FALSE, FALSE); +} + +GST_END_TEST; + +GST_START_TEST (test_start_time_zero_live_drop_3) +{ + run_test_start_time (FALSE, TRUE, FALSE); +} + +GST_END_TEST; + +GST_START_TEST (test_start_time_zero_live_drop_3_unlinked_1) +{ + run_test_start_time (FALSE, TRUE, TRUE); +} + +GST_END_TEST; + +GST_START_TEST (test_start_time_first_live_drop_0) +{ + run_test_start_time (TRUE, FALSE, FALSE); +} + +GST_END_TEST; + +GST_START_TEST (test_start_time_first_live_drop_3) +{ + run_test_start_time (TRUE, TRUE, FALSE); +} + +GST_END_TEST; + +GST_START_TEST (test_start_time_first_live_drop_3_unlinked_1) +{ + run_test_start_time (TRUE, TRUE, TRUE); +} + +GST_END_TEST; + static Suite * compositor_suite (void) { @@ -1446,6 +1606,12 @@ compositor_suite (void) tcase_add_test (tc_chain, test_segment_base_handling); tcase_add_test (tc_chain, test_obscured_skipped); tcase_add_test (tc_chain, test_ignore_eos); + tcase_add_test (tc_chain, test_start_time_zero_live_drop_0); + tcase_add_test (tc_chain, test_start_time_zero_live_drop_3); + tcase_add_test (tc_chain, test_start_time_zero_live_drop_3_unlinked_1); + tcase_add_test (tc_chain, test_start_time_first_live_drop_0); + tcase_add_test (tc_chain, test_start_time_first_live_drop_3); + tcase_add_test (tc_chain, test_start_time_first_live_drop_3_unlinked_1); /* Use a longer timeout */ #ifdef HAVE_VALGRIND From 7e67bcba2f1c4cdfa2cb165d0cad17d0ea44d636 Mon Sep 17 00:00:00 2001 From: Luis de Bethencourt Date: Wed, 29 Jul 2015 17:48:58 +0100 Subject: [PATCH 208/381] glstereomix: remove redundant initialization v is initialized in the for loop init, no need to do it twice. Removing first initialization. --- ext/gl/gstglstereomix.c | 1 - 1 file changed, 1 deletion(-) diff --git a/ext/gl/gstglstereomix.c b/ext/gl/gstglstereomix.c index 1422823f77..731569a67b 100644 --- a/ext/gl/gstglstereomix.c +++ b/ext/gl/gstglstereomix.c @@ -574,7 +574,6 @@ gst_gl_stereo_mix_process_frames (GstGLStereoMix * mixer, GPtrArray * frames) return FALSE; converted_buffer = mixer->primary_out; - v = 0; n = gst_buffer_n_memory (converted_buffer); g_assert (n == GST_VIDEO_INFO_N_PLANES (out_info) * views); for (v = 0; v < views; v++) { From db0a5ece027e7d0640be73a48b32ba256ace418e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Olivier=20Cr=C3=AAte?= Date: Fri, 6 Mar 2015 21:32:04 -0500 Subject: [PATCH 209/381] videoaggregator: Remove broken _clip vfunc It never does anything. https://bugzilla.gnome.org/show_bug.cgi?id=745768 --- gst-libs/gst/video/gstvideoaggregator.c | 63 ------------------------- 1 file changed, 63 deletions(-) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index 167c09983e..b8d29eeb9e 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -1620,68 +1620,6 @@ gst_videoaggregator_src_event (GstAggregator * agg, GstEvent * event) event); } -static GstFlowReturn -gst_videoaggregator_sink_clip (GstAggregator * agg, - GstAggregatorPad * bpad, GstBuffer * buf, GstBuffer ** outbuf) -{ - GstVideoAggregatorPad *pad = GST_VIDEO_AGGREGATOR_PAD (bpad); - GstClockTime start_time, end_time; - GstBuffer *pbuf; - - start_time = GST_BUFFER_TIMESTAMP (buf); - if (start_time == -1) { - GST_WARNING_OBJECT (pad, "Timestamped buffers required!"); - gst_buffer_unref (buf); - *outbuf = NULL; - return GST_FLOW_ERROR; - } - - end_time = GST_BUFFER_DURATION (buf); - if (end_time == -1 && GST_VIDEO_INFO_FPS_N (&pad->info) != 0) - end_time = - gst_util_uint64_scale_int_round (GST_SECOND, - GST_VIDEO_INFO_FPS_D (&pad->info), GST_VIDEO_INFO_FPS_N (&pad->info)); - if (end_time == -1) { - *outbuf = buf; - return GST_FLOW_OK; - } - - GST_OBJECT_LOCK (bpad); - - start_time = MAX (start_time, bpad->segment.start); - start_time = - gst_segment_to_running_time (&bpad->segment, GST_FORMAT_TIME, start_time); - - end_time += GST_BUFFER_TIMESTAMP (buf); - if (bpad->segment.stop != -1) - end_time = MIN (end_time, bpad->segment.stop); - end_time = - gst_segment_to_running_time (&bpad->segment, GST_FORMAT_TIME, end_time); - - /* Convert to the output segment rate */ - if (ABS (agg->segment.rate) != 1.0) { - end_time *= ABS (agg->segment.rate); - } - - pbuf = gst_aggregator_pad_get_buffer (bpad); - if (pbuf != NULL) { - gst_buffer_unref (pbuf); - - if (end_time < pad->priv->end_time) { - gst_buffer_unref (buf); - *outbuf = NULL; - goto done; - } - } - - *outbuf = buf; - -done: - - GST_OBJECT_UNLOCK (bpad); - return GST_FLOW_OK; -} - static GstFlowReturn gst_videoaggregator_flush (GstAggregator * agg) { @@ -2025,7 +1963,6 @@ gst_videoaggregator_class_init (GstVideoAggregatorClass * klass) agg_class->sink_query = gst_videoaggregator_sink_query; agg_class->sink_event = gst_videoaggregator_sink_event; agg_class->flush = gst_videoaggregator_flush; - agg_class->clip = gst_videoaggregator_sink_clip; agg_class->aggregate = gst_videoaggregator_aggregate; agg_class->src_event = gst_videoaggregator_src_event; agg_class->src_query = gst_videoaggregator_src_query; From 81a78449d5b408ad719e58e3ee818316169e6b7f Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Mon, 3 Aug 2015 19:48:16 +1000 Subject: [PATCH 210/381] glvideomixer: swap control binding proxy The ref_object and object parameters were the wrong way around. For the typical use case where an application is setting a GstControlBinding on the returned ghost pad: 1. our control binding would be removed when the new one was set 2. sync_values calls were not being forwarded from the internal pad to the ghost pad. If an application attempts to perform other control binding operations (get_* family of functions) on the internal pad, they will also be forwarded to the ghost pad where a possible GstControlBinding will provide the necessary values. --- ext/gl/gstglvideomixer.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ext/gl/gstglvideomixer.c b/ext/gl/gstglvideomixer.c index 25897d8731..d17a744945 100644 --- a/ext/gl/gstglvideomixer.c +++ b/ext/gl/gstglvideomixer.c @@ -327,10 +327,10 @@ _create_video_mixer_input (GstGLMixerBin * self, GstPad * mixer_pad) gst_object_unref (input); return NULL; } -#define ADD_PROXY_CONTROL_BINDING(prop) \ - cb = gst_gl_mixer_control_binding_proxy_new (GST_OBJECT (input), \ - G_STRINGIFY (prop), GST_OBJECT (mixer_pad), G_STRINGIFY (prop)); \ - gst_object_add_control_binding (GST_OBJECT (input), cb) +#define ADD_PROXY_CONTROL_BINDING(prop) \ + cb = gst_gl_mixer_control_binding_proxy_new (GST_OBJECT (mixer_pad), \ + G_STRINGIFY (prop), GST_OBJECT (input), G_STRINGIFY (prop)); \ + gst_object_add_control_binding (GST_OBJECT (mixer_pad), cb) ADD_PROXY_CONTROL_BINDING (zorder); ADD_PROXY_CONTROL_BINDING (xpos); From 708936de490b6afe55d32771629dcc8c4b3e89fc Mon Sep 17 00:00:00 2001 From: Ben Browitt Date: Thu, 20 Aug 2015 14:11:56 +0300 Subject: [PATCH 211/381] videoaggregator: Always set the pad's buffer_vinfo when storing a buffer Otherwise it might be unset, and then the buffer is used and gst_video_frame_map() will crash because of invalid video-info. https://bugzilla.gnome.org/show_bug.cgi?id=753805 --- gst-libs/gst/video/gstvideoaggregator.c | 1 + 1 file changed, 1 insertion(+) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index b8d29eeb9e..fe41e022fc 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -1025,6 +1025,7 @@ gst_videoaggregator_fill_queues (GstVideoAggregator * vagg, GST_DEBUG_OBJECT (pad, "buffer duration is -1, start_time < " "output_start_time. Discarding old buffer"); gst_buffer_replace (&pad->buffer, buf); + pad->buffer_vinfo = *vinfo; gst_buffer_unref (buf); gst_aggregator_pad_drop_buffer (bpad); need_more_data = TRUE; From f12ef34410afcdf727f5182328b745050b5490f3 Mon Sep 17 00:00:00 2001 From: Nirbheek Chauhan Date: Wed, 26 Aug 2015 15:40:16 +0530 Subject: [PATCH 212/381] compositor: Actually use the output resolution for clamping The obscured check in compositor was using the dimensions of the pad to clamp the h/w of the pad instead of the output resolution, and was doing an incorrect calculation to do so. Fix that by simplifying the whole calculation by using corner coordinates. Also add a test for this bug which fell through the cracks, and just skip all the obscured tests if the pad's alpha is 0.0. https://bugzilla.gnome.org/show_bug.cgi?id=754107 --- gst/compositor/compositor.c | 58 ++++++++++++++++++++++--------- tests/check/elements/compositor.c | 11 ++++++ 2 files changed, 52 insertions(+), 17 deletions(-) diff --git a/gst/compositor/compositor.c b/gst/compositor/compositor.c index 2e8ab5cc56..9aa651f95c 100644 --- a/gst/compositor/compositor.c +++ b/gst/compositor/compositor.c @@ -350,6 +350,26 @@ is_rectangle_contained (GstVideoRectangle rect1, GstVideoRectangle rect2) return FALSE; } +static GstVideoRectangle +clamp_rectangle (guint x, guint y, guint w, guint h, guint outer_width, + guint outer_height) +{ + guint x2 = x + w; + guint y2 = y + h; + GstVideoRectangle clamped; + + /* Clamp the x/y coordinates of this frame to the output boundaries to cover + * the case where (say, with negative xpos/ypos or w/h greater than the output + * size) the non-obscured portion of the frame could be outside the bounds of + * the video itself and hence not visible at all */ + clamped.x = CLAMP (x, 0, outer_width); + clamped.y = CLAMP (y, 0, outer_height); + clamped.w = CLAMP (x2, 0, outer_width) - clamped.x; + clamped.h = CLAMP (y2, 0, outer_height) - clamped.y; + + return clamped; +} + static gboolean gst_compositor_pad_prepare_frame (GstVideoAggregatorPad * pad, GstVideoAggregator * vagg) @@ -451,15 +471,21 @@ gst_compositor_pad_prepare_frame (GstVideoAggregatorPad * pad, g_free (wanted_colorimetry); } - /* Clamp the x/y coordinates of this frame to the video boundaries to cover - * the case where (say, with negative xpos/ypos) the non-obscured portion of - * the frame could be outside the bounds of the video itself and hence not - * visible at all */ - frame_rect.x = CLAMP (cpad->xpos, 0, GST_VIDEO_INFO_WIDTH (&vagg->info)); - frame_rect.y = CLAMP (cpad->ypos, 0, GST_VIDEO_INFO_HEIGHT (&vagg->info)); - /* Clamp the width/height to the frame boundaries as well */ - frame_rect.w = MAX (width - frame_rect.x, 0); - frame_rect.h = MAX (height - frame_rect.y, 0); + if (cpad->alpha == 0.0) { + GST_DEBUG_OBJECT (vagg, "Pad has alpha 0.0, not converting frame"); + converted_frame = NULL; + goto done; + } + + frame_rect = clamp_rectangle (cpad->xpos, cpad->ypos, width, height, + GST_VIDEO_INFO_WIDTH (&vagg->info), GST_VIDEO_INFO_HEIGHT (&vagg->info)); + + if (frame_rect.w == 0 || frame_rect.h == 0) { + GST_DEBUG_OBJECT (vagg, "Resulting frame is zero-width or zero-height " + "(w: %i, h: %i), skipping", frame_rect.w, frame_rect.h); + converted_frame = NULL; + goto done; + } GST_OBJECT_LOCK (vagg); /* Check if this frame is obscured by a higher-zorder frame @@ -488,8 +514,12 @@ gst_compositor_pad_prepare_frame (GstVideoAggregatorPad * pad, !GST_VIDEO_INFO_HAS_ALPHA (&pad2->info) && is_rectangle_contained (frame_rect, frame2_rect)) { frame_obscured = TRUE; - GST_DEBUG_OBJECT (pad, "Obscured by %s, skipping frame", - GST_PAD_NAME (pad2)); + GST_DEBUG_OBJECT (pad, "%ix%i@(%i,%i) obscured by %s %ix%i@(%i,%i) " + "in output of size %ix%i; skipping frame", frame_rect.w, frame_rect.h, + frame_rect.x, frame_rect.y, GST_PAD_NAME (pad2), frame2_rect.w, + frame2_rect.h, frame2_rect.x, frame2_rect.y, + GST_VIDEO_INFO_WIDTH (&vagg->info), + GST_VIDEO_INFO_HEIGHT (&vagg->info)); break; } } @@ -500,12 +530,6 @@ gst_compositor_pad_prepare_frame (GstVideoAggregatorPad * pad, goto done; } - if (cpad->alpha == 0.0) { - GST_DEBUG_OBJECT (vagg, "Pad has alpha 0.0, not converting frame"); - converted_frame = NULL; - goto done; - } - frame = g_slice_new0 (GstVideoFrame); if (!gst_video_frame_map (frame, &pad->buffer_vinfo, pad->buffer, diff --git a/tests/check/elements/compositor.c b/tests/check/elements/compositor.c index a949d3828c..dc1a58f031 100644 --- a/tests/check/elements/compositor.c +++ b/tests/check/elements/compositor.c @@ -1315,6 +1315,17 @@ GST_START_TEST (test_obscured_skipped) xpos0 = ypos0 = xpos1 = ypos1 = 0; buffer_mapped = FALSE; + xpos1 = ypos1 = 0; + xpos0 = ypos0 = width0 = height0 = width1 = height1 = 10; + out_width = out_height = 20; + GST_INFO ("testing bug 754107"); + _test_obscured (caps_str, xpos0, ypos0, width0, height0, alpha0, xpos1, ypos1, + width1, height1, alpha1, out_width, out_height); + fail_unless (buffer_mapped == TRUE); + xpos0 = ypos0 = xpos1 = ypos1 = width0 = height0 = width1 = height1 = 0; + out_width = out_height = 0; + buffer_mapped = FALSE; + xpos0 = ypos0 = 10000; out_width = 320; out_height = 240; From e4b584b3e53e40c150c0d62a3b3a96d0346c86c4 Mon Sep 17 00:00:00 2001 From: Luis de Bethencourt Date: Fri, 28 Aug 2015 15:21:11 +0100 Subject: [PATCH 213/381] compositor: remove check for below zero for unsigned value CLAMP checks both if value is '< 0' and '> max'. Value will never be a negative number since it is an unsigned integer. Removing that check and only checking if it is bigger than max by using MIN(). CID 1320707 --- gst/compositor/compositor.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/gst/compositor/compositor.c b/gst/compositor/compositor.c index 9aa651f95c..66927ba3a9 100644 --- a/gst/compositor/compositor.c +++ b/gst/compositor/compositor.c @@ -362,10 +362,10 @@ clamp_rectangle (guint x, guint y, guint w, guint h, guint outer_width, * the case where (say, with negative xpos/ypos or w/h greater than the output * size) the non-obscured portion of the frame could be outside the bounds of * the video itself and hence not visible at all */ - clamped.x = CLAMP (x, 0, outer_width); - clamped.y = CLAMP (y, 0, outer_height); - clamped.w = CLAMP (x2, 0, outer_width) - clamped.x; - clamped.h = CLAMP (y2, 0, outer_height) - clamped.y; + clamped.x = MIN (x, outer_width); + clamped.y = MIN (y, outer_height); + clamped.w = MIN (x2, outer_width) - clamped.x; + clamped.h = MIN (y2, outer_height) - clamped.y; return clamped; } From 8c06c85d83dfd3ca7dfc286366d8865e193bac83 Mon Sep 17 00:00:00 2001 From: Nirbheek Chauhan Date: Sat, 29 Aug 2015 16:51:08 +0530 Subject: [PATCH 214/381] compositor: variables in clamp_rectangle() should be signed x/y/w/h are signed integers. As can be seen in GstCompositorPad. The prototype for clamp_rectangle was wrong. This commit reverts the change and fixes the prototype. This reverts commit bca444ea4a84c39e9989681f892f6e4cb2033cf9. --- gst/compositor/compositor.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/gst/compositor/compositor.c b/gst/compositor/compositor.c index 66927ba3a9..64d05ac180 100644 --- a/gst/compositor/compositor.c +++ b/gst/compositor/compositor.c @@ -351,21 +351,21 @@ is_rectangle_contained (GstVideoRectangle rect1, GstVideoRectangle rect2) } static GstVideoRectangle -clamp_rectangle (guint x, guint y, guint w, guint h, guint outer_width, +clamp_rectangle (gint x, gint y, gint w, gint h, guint outer_width, guint outer_height) { - guint x2 = x + w; - guint y2 = y + h; + gint x2 = x + w; + gint y2 = y + h; GstVideoRectangle clamped; /* Clamp the x/y coordinates of this frame to the output boundaries to cover * the case where (say, with negative xpos/ypos or w/h greater than the output * size) the non-obscured portion of the frame could be outside the bounds of * the video itself and hence not visible at all */ - clamped.x = MIN (x, outer_width); - clamped.y = MIN (y, outer_height); - clamped.w = MIN (x2, outer_width) - clamped.x; - clamped.h = MIN (y2, outer_height) - clamped.y; + clamped.x = CLAMP (x, 0, outer_width); + clamped.y = CLAMP (y, 0, outer_height); + clamped.w = CLAMP (x2, 0, outer_width) - clamped.x; + clamped.h = CLAMP (y2, 0, outer_height) - clamped.y; return clamped; } From 4eea087f68a15e6fffa9fe95e0ee634fb3f0d24f Mon Sep 17 00:00:00 2001 From: Thiago Santos Date: Wed, 2 Sep 2015 19:16:56 -0300 Subject: [PATCH 215/381] videoaggregator: lift restriction of changing pixel-aspect-ratio The videoaggregator can convert PAR, there is no reason for restricting it. https://bugzilla.gnome.org/show_bug.cgi?id=754291 --- gst-libs/gst/video/gstvideoaggregator.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index fe41e022fc..80b10d4452 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -813,9 +813,7 @@ gst_videoaggregator_pad_sink_setcaps (GstPad * pad, GstObject * parent, GST_VIDEO_AGGREGATOR_LOCK (vagg); if (GST_VIDEO_INFO_FORMAT (&vagg->info) != GST_VIDEO_FORMAT_UNKNOWN) { - if (GST_VIDEO_INFO_PAR_N (&vagg->info) != GST_VIDEO_INFO_PAR_N (&info) - || GST_VIDEO_INFO_PAR_D (&vagg->info) != GST_VIDEO_INFO_PAR_D (&info) || - GST_VIDEO_INFO_INTERLACE_MODE (&vagg->info) != + if (GST_VIDEO_INFO_INTERLACE_MODE (&vagg->info) != GST_VIDEO_INFO_INTERLACE_MODE (&info)) { GST_ERROR_OBJECT (pad, "got input caps %" GST_PTR_FORMAT ", but " "current caps are %" From f1f8e465f27d41f55524fa5c6cfc90d798a310bd Mon Sep 17 00:00:00 2001 From: Nirbheek Chauhan Date: Sat, 5 Sep 2015 01:50:41 +0530 Subject: [PATCH 216/381] compositor: Ensure all arguments to CLAMP are signed int If any of the arguments to CLAMP are unsigned integers, the comparison causes an automatic conversion of the signed int to unsigned, which causes -1 to become UINT_MAX and get clamped to the high value of the CLAMP instead of 0. See 716 at http://c0x.coding-guidelines.com/6.3.1.8.html Also add a test for this. https://bugzilla.gnome.org/show_bug.cgi?id=754576 --- gst/compositor/compositor.c | 4 ++-- tests/check/elements/compositor.c | 11 +++++++++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/gst/compositor/compositor.c b/gst/compositor/compositor.c index 64d05ac180..d015d85e10 100644 --- a/gst/compositor/compositor.c +++ b/gst/compositor/compositor.c @@ -351,8 +351,8 @@ is_rectangle_contained (GstVideoRectangle rect1, GstVideoRectangle rect2) } static GstVideoRectangle -clamp_rectangle (gint x, gint y, gint w, gint h, guint outer_width, - guint outer_height) +clamp_rectangle (gint x, gint y, gint w, gint h, gint outer_width, + gint outer_height) { gint x2 = x + w; gint y2 = y + h; diff --git a/tests/check/elements/compositor.c b/tests/check/elements/compositor.c index dc1a58f031..ccd68a1970 100644 --- a/tests/check/elements/compositor.c +++ b/tests/check/elements/compositor.c @@ -1326,6 +1326,17 @@ GST_START_TEST (test_obscured_skipped) out_width = out_height = 0; buffer_mapped = FALSE; + xpos1 = -1; + xpos0 = ypos0 = width0 = height0 = width1 = height1 = 10; + out_width = out_height = 20; + GST_INFO ("testing bug 754576"); + _test_obscured (caps_str, xpos0, ypos0, width0, height0, alpha0, xpos1, ypos1, + width1, height1, alpha1, out_width, out_height); + fail_unless (buffer_mapped == TRUE); + xpos0 = xpos1 = ypos1 = width0 = height0 = width1 = height1 = 0; + out_width = out_height = 0; + buffer_mapped = FALSE; + xpos0 = ypos0 = 10000; out_width = 320; out_height = 240; From ac503aee7d958672aa909e0554b4aee5258cd142 Mon Sep 17 00:00:00 2001 From: Thiago Santos Date: Wed, 9 Sep 2015 19:51:18 -0300 Subject: [PATCH 217/381] videoaggregator: fix caps query to properly handle alpha formats Only accept alpha if downstream has alpha as well. It could theoretically accept alpha unconditionally if blending is properly implemented for handle it but at the moment this is a missing feature. Improves the caps query by also comparing with the template caps to filter by what the subclass supports. https://bugzilla.gnome.org/show_bug.cgi?id=754465 --- gst-libs/gst/video/gstvideoaggregator.c | 211 ++++++++++++++++++++++-- gst-libs/gst/video/gstvideoaggregator.h | 2 + 2 files changed, 200 insertions(+), 13 deletions(-) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index 80b10d4452..053e81be6b 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -76,6 +76,7 @@ struct _GstVideoAggregatorPadPrivate GstClockTime end_time; }; + G_DEFINE_TYPE (GstVideoAggregatorPad, gst_videoaggregator_pad, GST_TYPE_AGGREGATOR_PAD); @@ -429,9 +430,44 @@ struct _GstVideoAggregatorPrivate gboolean live; }; -G_DEFINE_ABSTRACT_TYPE_WITH_CODE (GstVideoAggregator, gst_videoaggregator, - GST_TYPE_AGGREGATOR, G_IMPLEMENT_INTERFACE (GST_TYPE_CHILD_PROXY, - gst_videoaggregator_child_proxy_init)); +/* Can't use the G_DEFINE_TYPE macros because we need the + * videoaggregator class in the _init to be able to set + * the sink pad non-alpha caps. Using the G_DEFINE_TYPE there + * seems to be no way of getting the real class being initialized */ +static void gst_videoaggregator_init (GstVideoAggregator * self, + GstVideoAggregatorClass * klass); +static void gst_videoaggregator_class_init (GstVideoAggregatorClass * klass); +static gpointer gst_videoaggregator_parent_class = NULL; +static gint GstVideoAggregator_private_offset; + +_G_DEFINE_TYPE_EXTENDED_CLASS_INIT (GstVideoAggregator, gst_videoaggregator); + +G_GNUC_UNUSED static inline gpointer +gst_videoaggregator_get_instance_private (const GstVideoAggregator * self) +{ + return (G_STRUCT_MEMBER_P (self, GstVideoAggregator_private_offset)); +} + +GType +gst_videoaggregator_get_type (void) +{ + static volatile gsize g_define_type_id_volatile = 0; + if (g_once_init_enter (&g_define_type_id_volatile)) { + GType g_define_type_id = g_type_register_static_simple (GST_TYPE_AGGREGATOR, + g_intern_static_string ("GstVideoAggregator"), + sizeof (GstVideoAggregatorClass), + (GClassInitFunc) gst_videoaggregator_class_intern_init, + sizeof (GstVideoAggregator), + (GInstanceInitFunc) gst_videoaggregator_init, + (GTypeFlags) G_TYPE_FLAG_ABSTRACT); + { + G_IMPLEMENT_INTERFACE (GST_TYPE_CHILD_PROXY, + gst_videoaggregator_child_proxy_init); + } + g_once_init_leave (&g_define_type_id_volatile, g_define_type_id); + } + return g_define_type_id_volatile; +} static void gst_videoaggreagator_find_best_format (GstVideoAggregator * vagg, @@ -833,27 +869,80 @@ beach: return ret; } +static gboolean +gst_videoaggregator_caps_has_alpha (GstCaps * caps) +{ + guint size = gst_caps_get_size (caps); + guint i; + + for (i = 0; i < size; i++) { + GstStructure *s = gst_caps_get_structure (caps, i); + const GValue *formats = gst_structure_get_value (s, "format"); + + if (formats) { + const GstVideoFormatInfo *info; + + if (GST_VALUE_HOLDS_LIST (formats)) { + guint list_size = gst_value_list_get_size (formats); + guint index; + + for (index = 0; index < list_size; index++) { + const GValue *list_item = gst_value_list_get_value (formats, index); + info = + gst_video_format_get_info (gst_video_format_from_string + (g_value_get_string (list_item))); + if (GST_VIDEO_FORMAT_INFO_HAS_ALPHA (info)) + return TRUE; + } + + } else if (G_VALUE_HOLDS_STRING (formats)) { + info = + gst_video_format_get_info (gst_video_format_from_string + (g_value_get_string (formats))); + if (GST_VIDEO_FORMAT_INFO_HAS_ALPHA (info)) + return TRUE; + + } else { + g_assert_not_reached (); + GST_WARNING ("Unexpected type for video 'format' field: %s", + G_VALUE_TYPE_NAME (formats)); + } + + } else { + return TRUE; + } + } + return FALSE; +} + static GstCaps * gst_videoaggregator_pad_sink_getcaps (GstPad * pad, GstVideoAggregator * vagg, GstCaps * filter) { GstCaps *srccaps; - GstCaps *template_caps; + GstCaps *template_caps, *sink_template_caps; GstCaps *returned_caps; GstStructure *s; - gboolean had_current_caps = TRUE; gint i, n; GstAggregator *agg = GST_AGGREGATOR (vagg); + GstPad *srcpad = GST_PAD (agg->srcpad); + gboolean has_alpha; - template_caps = gst_pad_get_pad_template_caps (GST_PAD (agg->srcpad)); + template_caps = gst_pad_get_pad_template_caps (srcpad); - srccaps = gst_pad_get_current_caps (GST_PAD (agg->srcpad)); + GST_DEBUG_OBJECT (pad, "Get caps with filter: %" GST_PTR_FORMAT, filter); + + srccaps = gst_pad_get_current_caps (srcpad); if (srccaps == NULL) { - had_current_caps = FALSE; - srccaps = template_caps; + srccaps = gst_pad_peer_query_caps (srcpad, template_caps); + GST_DEBUG_OBJECT (pad, "No output caps, using possible formats: %" + GST_PTR_FORMAT, srccaps); + } else { + GST_DEBUG_OBJECT (pad, "Using output caps: %" GST_PTR_FORMAT, srccaps); } srccaps = gst_caps_make_writable (srccaps); + has_alpha = gst_videoaggregator_caps_has_alpha (srccaps); n = gst_caps_get_size (srccaps); for (i = 0; i < n; i++) { @@ -873,8 +962,23 @@ gst_videoaggregator_pad_sink_getcaps (GstPad * pad, GstVideoAggregator * vagg, returned_caps = srccaps; } - if (had_current_caps) - gst_caps_unref (template_caps); + if (has_alpha) { + sink_template_caps = gst_pad_get_pad_template_caps (pad); + } else { + GstVideoAggregatorClass *klass = GST_VIDEO_AGGREGATOR_GET_CLASS (vagg); + sink_template_caps = gst_caps_ref (klass->sink_non_alpha_caps); + } + + { + GstCaps *intersect = gst_caps_intersect (returned_caps, sink_template_caps); + gst_caps_unref (returned_caps); + returned_caps = intersect; + } + + gst_caps_unref (template_caps); + gst_caps_unref (sink_template_caps); + + GST_DEBUG_OBJECT (pad, "Returning caps: %" GST_PTR_FORMAT, returned_caps); return returned_caps; } @@ -1974,9 +2078,83 @@ gst_videoaggregator_class_init (GstVideoAggregatorClass * klass) g_type_class_ref (GST_TYPE_VIDEO_AGGREGATOR_PAD); } -static void -gst_videoaggregator_init (GstVideoAggregator * vagg) +static inline GstCaps * +_get_non_alpha_caps_from_template (GstVideoAggregatorClass * klass) { + GstCaps *result; + GstCaps *templatecaps; + guint i, size; + + templatecaps = + gst_pad_template_get_caps (gst_element_class_get_pad_template + (GST_ELEMENT_CLASS (klass), "sink_%u")); + + size = gst_caps_get_size (templatecaps); + result = gst_caps_new_empty (); + for (i = 0; i < size; i++) { + GstStructure *s = gst_caps_get_structure (templatecaps, i); + const GValue *formats = gst_structure_get_value (s, "format"); + GValue new_formats = { 0, }; + gboolean has_format = FALSE; + + /* FIXME what to do if formats are missing? */ + if (formats) { + const GstVideoFormatInfo *info; + + if (GST_VALUE_HOLDS_LIST (formats)) { + guint list_size = gst_value_list_get_size (formats); + guint index; + + g_value_init (&new_formats, GST_TYPE_LIST); + + for (index = 0; index < list_size; index++) { + const GValue *list_item = gst_value_list_get_value (formats, index); + + info = + gst_video_format_get_info (gst_video_format_from_string + (g_value_get_string (list_item))); + if (!GST_VIDEO_FORMAT_INFO_HAS_ALPHA (info)) { + has_format = TRUE; + gst_value_list_append_value (&new_formats, list_item); + } + } + + } else if (G_VALUE_HOLDS_STRING (formats)) { + info = + gst_video_format_get_info (gst_video_format_from_string + (g_value_get_string (formats))); + if (!GST_VIDEO_FORMAT_INFO_HAS_ALPHA (info)) { + has_format = TRUE; + gst_value_init_and_copy (&new_formats, formats); + } + + } else { + g_assert_not_reached (); + GST_WARNING ("Unexpected type for video 'format' field: %s", + G_VALUE_TYPE_NAME (formats)); + } + + if (has_format) { + s = gst_structure_copy (s); + gst_structure_take_value (s, "format", &new_formats); + gst_caps_append_structure (result, s); + } + + } + } + + gst_caps_unref (templatecaps); + + return result; +} + +static GMutex sink_caps_mutex; + +static void +gst_videoaggregator_init (GstVideoAggregator * vagg, + GstVideoAggregatorClass * klass) +{ + vagg->priv = G_TYPE_INSTANCE_GET_PRIVATE (vagg, GST_TYPE_VIDEO_AGGREGATOR, GstVideoAggregatorPrivate); @@ -1984,6 +2162,13 @@ gst_videoaggregator_init (GstVideoAggregator * vagg) vagg->priv->current_caps = NULL; g_mutex_init (&vagg->priv->lock); + /* initialize variables */ + g_mutex_lock (&sink_caps_mutex); + if (klass->sink_non_alpha_caps == NULL) { + klass->sink_non_alpha_caps = _get_non_alpha_caps_from_template (klass); + } + g_mutex_unlock (&sink_caps_mutex); + gst_videoaggregator_reset (vagg); } diff --git a/gst-libs/gst/video/gstvideoaggregator.h b/gst-libs/gst/video/gstvideoaggregator.h index b3c5c53e78..f95d0d2674 100644 --- a/gst-libs/gst/video/gstvideoaggregator.h +++ b/gst-libs/gst/video/gstvideoaggregator.h @@ -111,6 +111,8 @@ struct _GstVideoAggregatorClass gboolean preserve_update_caps_result; + GstCaps *sink_non_alpha_caps; + /* < private > */ gpointer _gst_reserved[GST_PADDING_LARGE]; }; From 895e71e810650e5d353ed60cf67e69fd0d02e2cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Fri, 11 Sep 2015 12:22:51 +0200 Subject: [PATCH 218/381] videoaggregator: Fix mixup of running times and segment positions We have to queue buffers based on their running time, not based on the segment position. Also return running time from GstAggregator::get_next_time() instead of a segment position, as required by the API. Also only update the segment position after we pushed a buffer, otherwise we're going to push down a segment event with the next position already. https://bugzilla.gnome.org/show_bug.cgi?id=753196 --- gst-libs/gst/video/gstvideoaggregator.c | 90 +++++++++++++++++-------- 1 file changed, 61 insertions(+), 29 deletions(-) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index 053e81be6b..09347c37d8 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -1063,15 +1063,16 @@ gst_videoaggregator_reset (GstVideoAggregator * vagg) #define GST_FLOW_NEEDS_DATA GST_FLOW_CUSTOM_ERROR static gint gst_videoaggregator_fill_queues (GstVideoAggregator * vagg, - GstClockTime output_start_time, GstClockTime output_end_time) + GstClockTime output_start_running_time, + GstClockTime output_end_running_time) { GstAggregator *agg = GST_AGGREGATOR (vagg); GList *l; gboolean eos = TRUE; gboolean need_more_data = FALSE; - /* get a set of buffers into pad->buffer that are within output_start_time - * and output_end_time taking into account finished and unresponsive pads */ + /* get a set of buffers into pad->buffer that are within output_start_running_time + * and output_end_running_time taking into account finished and unresponsive pads */ GST_OBJECT_LOCK (vagg); for (l = GST_ELEMENT (vagg)->sinkpads; l; l = l->next) { @@ -1112,20 +1113,20 @@ gst_videoaggregator_fill_queues (GstVideoAggregator * vagg, start_time = gst_segment_to_running_time (&segment, GST_FORMAT_TIME, start_time); - if (start_time >= output_end_time) { + if (start_time >= output_end_running_time) { if (pad->buffer) { GST_DEBUG_OBJECT (pad, "buffer duration is -1, start_time >= " - "output_end_time. Keeping previous buffer"); + "output_end_running_time. Keeping previous buffer"); } else { GST_DEBUG_OBJECT (pad, "buffer duration is -1, start_time >= " - "output_end_time. No previous buffer, need more data"); + "output_end_running_time. No previous buffer, need more data"); need_more_data = TRUE; } gst_buffer_unref (buf); continue; - } else if (start_time < output_start_time) { + } else if (start_time < output_start_running_time) { GST_DEBUG_OBJECT (pad, "buffer duration is -1, start_time < " - "output_start_time. Discarding old buffer"); + "output_start_running_time. Discarding old buffer"); gst_buffer_replace (&pad->buffer, buf); pad->buffer_vinfo = *vinfo; gst_buffer_unref (buf); @@ -1181,8 +1182,8 @@ gst_videoaggregator_fill_queues (GstVideoAggregator * vagg, GST_TRACE_OBJECT (pad, "dealing with buffer %p start %" GST_TIME_FORMAT " end %" GST_TIME_FORMAT " out start %" GST_TIME_FORMAT " out end %" GST_TIME_FORMAT, buf, GST_TIME_ARGS (start_time), - GST_TIME_ARGS (end_time), GST_TIME_ARGS (output_start_time), - GST_TIME_ARGS (output_end_time)); + GST_TIME_ARGS (end_time), GST_TIME_ARGS (output_start_running_time), + GST_TIME_ARGS (output_end_running_time)); if (pad->priv->end_time != -1 && pad->priv->end_time > end_time) { GST_DEBUG_OBJECT (pad, "Buffer from the past, dropping"); @@ -1191,7 +1192,8 @@ gst_videoaggregator_fill_queues (GstVideoAggregator * vagg, continue; } - if (end_time >= output_start_time && start_time < output_end_time) { + if (end_time >= output_start_running_time + && start_time < output_end_running_time) { GST_DEBUG_OBJECT (pad, "Taking new buffer with start time %" GST_TIME_FORMAT, GST_TIME_ARGS (start_time)); @@ -1203,7 +1205,7 @@ gst_videoaggregator_fill_queues (GstVideoAggregator * vagg, gst_buffer_unref (buf); gst_aggregator_pad_drop_buffer (bpad); eos = FALSE; - } else if (start_time >= output_end_time) { + } else if (start_time >= output_end_running_time) { GST_DEBUG_OBJECT (pad, "Keeping buffer until %" GST_TIME_FORMAT, GST_TIME_ARGS (start_time)); gst_buffer_unref (buf); @@ -1216,7 +1218,7 @@ gst_videoaggregator_fill_queues (GstVideoAggregator * vagg, GST_DEBUG_OBJECT (pad, "replacing old buffer with a newer buffer, start %" GST_TIME_FORMAT " out end %" GST_TIME_FORMAT, GST_TIME_ARGS (start_time), - GST_TIME_ARGS (output_end_time)); + GST_TIME_ARGS (output_end_running_time)); gst_buffer_unref (buf); gst_aggregator_pad_drop_buffer (bpad); @@ -1231,7 +1233,7 @@ gst_videoaggregator_fill_queues (GstVideoAggregator * vagg, } if (pad->priv->end_time != -1) { - if (pad->priv->end_time <= output_start_time) { + if (pad->priv->end_time <= output_start_running_time) { pad->priv->start_time = pad->priv->end_time = -1; if (is_eos) { GST_DEBUG ("I just need more data"); @@ -1392,10 +1394,22 @@ gst_videoaggregator_do_qos (GstVideoAggregator * vagg, GstClockTime timestamp) static GstClockTime gst_videoaggregator_get_next_time (GstAggregator * agg) { - if (agg->segment.position == -1) - return agg->segment.start; + GstClockTime next_time; + + GST_OBJECT_LOCK (agg); + if (agg->segment.position == -1 || agg->segment.position < agg->segment.start) + next_time = agg->segment.start; else - return agg->segment.position; + next_time = agg->segment.position; + + if (agg->segment.stop != -1 && next_time > agg->segment.stop) + next_time = agg->segment.stop; + + next_time = + gst_segment_to_running_time (&agg->segment, GST_FORMAT_TIME, next_time); + GST_OBJECT_UNLOCK (agg); + + return next_time; } static GstFlowReturn @@ -1456,6 +1470,7 @@ gst_videoaggregator_aggregate (GstAggregator * agg, gboolean timeout) { GstVideoAggregator *vagg = GST_VIDEO_AGGREGATOR (agg); GstClockTime output_start_time, output_end_time; + GstClockTime output_start_running_time, output_end_running_time; GstBuffer *outbuf = NULL; GstFlowReturn flow_ret; gint64 jitter; @@ -1469,31 +1484,42 @@ gst_videoaggregator_aggregate (GstAggregator * agg, gboolean timeout) goto unlock_and_return; } - output_start_time = gst_videoaggregator_get_next_time (agg); + output_start_time = agg->segment.position; + if (agg->segment.position == -1 || agg->segment.position < agg->segment.start) + output_start_time = agg->segment.start; + if (vagg->priv->nframes == 0) { vagg->priv->ts_offset = output_start_time; GST_DEBUG_OBJECT (vagg, "New ts offset %" GST_TIME_FORMAT, GST_TIME_ARGS (output_start_time)); } - if (GST_VIDEO_INFO_FPS_N (&vagg->info) == 0) + if (GST_VIDEO_INFO_FPS_N (&vagg->info) == 0) { output_end_time = -1; - else + } else { output_end_time = vagg->priv->ts_offset + gst_util_uint64_scale_round (vagg->priv->nframes + 1, GST_SECOND * GST_VIDEO_INFO_FPS_D (&vagg->info), GST_VIDEO_INFO_FPS_N (&vagg->info)); + } if (agg->segment.stop != -1) output_end_time = MIN (output_end_time, agg->segment.stop); + output_start_running_time = + gst_segment_to_running_time (&agg->segment, GST_FORMAT_TIME, + output_start_time); + output_end_running_time = + gst_segment_to_running_time (&agg->segment, GST_FORMAT_TIME, + output_end_time); + if (output_end_time == output_start_time) { flow_ret = GST_FLOW_EOS; } else { flow_ret = - gst_videoaggregator_fill_queues (vagg, output_start_time, - output_end_time); + gst_videoaggregator_fill_queues (vagg, output_start_running_time, + output_end_running_time); } if (flow_ret == GST_FLOW_NEEDS_DATA && !timeout) { @@ -1509,8 +1535,12 @@ gst_videoaggregator_aggregate (GstAggregator * agg, gboolean timeout) } GST_DEBUG_OBJECT (vagg, - "Producing buffer for %" GST_TIME_FORMAT " to %" GST_TIME_FORMAT, - GST_TIME_ARGS (output_start_time), GST_TIME_ARGS (output_end_time)); + "Producing buffer for %" GST_TIME_FORMAT " to %" GST_TIME_FORMAT + ", running time start %" GST_TIME_FORMAT ", running time end %" + GST_TIME_FORMAT, GST_TIME_ARGS (output_start_time), + GST_TIME_ARGS (output_end_time), + GST_TIME_ARGS (output_start_running_time), + GST_TIME_ARGS (output_end_running_time)); jitter = gst_videoaggregator_do_qos (vagg, output_start_time); if (jitter <= 0) { @@ -1526,8 +1556,7 @@ gst_videoaggregator_aggregate (GstAggregator * agg, gboolean timeout) msg = gst_message_new_qos (GST_OBJECT_CAST (vagg), vagg->priv->live, - gst_segment_to_running_time (&agg->segment, GST_FORMAT_TIME, - output_start_time), gst_segment_to_stream_time (&agg->segment, + output_start_running_time, gst_segment_to_stream_time (&agg->segment, GST_FORMAT_TIME, output_start_time), output_start_time, output_end_time - output_start_time); gst_message_set_qos_values (msg, jitter, vagg->priv->proportion, 1000000); @@ -1538,9 +1567,6 @@ gst_videoaggregator_aggregate (GstAggregator * agg, gboolean timeout) flow_ret = GST_FLOW_OK; } - agg->segment.position = output_end_time; - vagg->priv->nframes++; - GST_VIDEO_AGGREGATOR_UNLOCK (vagg); if (outbuf) { GST_DEBUG_OBJECT (vagg, @@ -1550,6 +1576,12 @@ gst_videoaggregator_aggregate (GstAggregator * agg, gboolean timeout) flow_ret = gst_aggregator_finish_buffer (agg, outbuf); } + + GST_VIDEO_AGGREGATOR_LOCK (vagg); + vagg->priv->nframes++; + agg->segment.position = output_end_time; + GST_VIDEO_AGGREGATOR_UNLOCK (vagg); + return flow_ret; done: From 78e07d02ebc4f03281b5ab65e0a5369700b91f59 Mon Sep 17 00:00:00 2001 From: Vineeth T M Date: Thu, 17 Sep 2015 13:35:02 +0900 Subject: [PATCH 219/381] gl: Fix GError leaks during failures https://bugzilla.gnome.org/show_bug.cgi?id=755140 --- ext/gl/gstglbasemixer.c | 1 + 1 file changed, 1 insertion(+) diff --git a/ext/gl/gstglbasemixer.c b/ext/gl/gstglbasemixer.c index f8c8d193d5..3bad76955d 100644 --- a/ext/gl/gstglbasemixer.c +++ b/ext/gl/gstglbasemixer.c @@ -496,6 +496,7 @@ context_error: { GST_ELEMENT_ERROR (mix, RESOURCE, NOT_FOUND, ("%s", error->message), (NULL)); + g_clear_error (&error); return FALSE; } } From d5f2742ad06275d0d3610da4cbde3f73fdf6f971 Mon Sep 17 00:00:00 2001 From: Jan Schmidt Date: Fri, 18 Sep 2015 00:20:13 +1000 Subject: [PATCH 220/381] Don't throw compiler warnings with G_DISABLE_ASSERT Disable code that warns about unused variables when G_DISABLE_ASSERT is defined, as it is in tarballs and pre-releases. --- ext/gl/gstglstereomix.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/ext/gl/gstglstereomix.c b/ext/gl/gstglstereomix.c index 731569a67b..b880d06357 100644 --- a/ext/gl/gstglstereomix.c +++ b/ext/gl/gstglstereomix.c @@ -503,7 +503,10 @@ gst_gl_stereo_mix_process_frames (GstGLStereoMix * mixer, GPtrArray * frames) GstVideoAggregator *vagg = GST_VIDEO_AGGREGATOR (mixer); GstBuffer *converted_buffer, *inbuf; GstVideoInfo *out_info = &vagg->info; - gint count = 0, n; + gint count = 0; +#ifndef G_DISABLE_ASSERT + gint n; +#endif gint v, views; gint valid_views = 0; @@ -574,8 +577,12 @@ gst_gl_stereo_mix_process_frames (GstGLStereoMix * mixer, GPtrArray * frames) return FALSE; converted_buffer = mixer->primary_out; + +#ifndef G_DISABLE_ASSERT n = gst_buffer_n_memory (converted_buffer); g_assert (n == GST_VIDEO_INFO_N_PLANES (out_info) * views); +#endif + for (v = 0; v < views; v++) { gst_buffer_add_video_meta_full (converted_buffer, v, GST_VIDEO_INFO_FORMAT (out_info), From 4d307bf4e2207d693f8d4a5b521267d5e0f47575 Mon Sep 17 00:00:00 2001 From: Nirbheek Chauhan Date: Sat, 29 Aug 2015 19:16:38 +0530 Subject: [PATCH 221/381] check: Add test for videoaggregator sinkpads being sorted by zorder https://bugzilla.gnome.org/show_bug.cgi?id=754285 --- tests/check/elements/compositor.c | 45 +++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/tests/check/elements/compositor.c b/tests/check/elements/compositor.c index ccd68a1970..77c65cfed1 100644 --- a/tests/check/elements/compositor.c +++ b/tests/check/elements/compositor.c @@ -1444,6 +1444,50 @@ GST_START_TEST (test_ignore_eos) GST_END_TEST; +/* Test that the GST_ELEMENT(vagg)->sinkpads GList is always sorted by zorder */ +GST_START_TEST (test_pad_z_order) +{ + GstElement *compositor; + GstPad *sinkpad1, *sinkpad2, *sinkpad3; + guint zorder1, zorder2; + GList *sinkpads; + + GST_INFO ("preparing test"); + + compositor = gst_element_factory_make ("compositor", NULL); + sinkpad1 = gst_element_get_request_pad (compositor, "sink_%u"); + sinkpad2 = gst_element_get_request_pad (compositor, "sink_%u"); + + /* Pads requested later have a higher z-order than earlier ones by default */ + g_object_get (sinkpad1, "zorder", &zorder1, NULL); + g_object_get (sinkpad2, "zorder", &zorder2, NULL); + ck_assert_int_gt (zorder2, zorder1); + sinkpads = GST_ELEMENT (compositor)->sinkpads; + ck_assert_ptr_eq (sinkpads->data, sinkpad1); + ck_assert_ptr_eq (sinkpads->next->data, sinkpad2); + + /* Make sinkpad1's zorder the largest, which should re-sort the sinkpads */ + g_object_set (sinkpad1, "zorder", zorder2 + 1, NULL); + sinkpads = GST_ELEMENT (compositor)->sinkpads; + ck_assert_ptr_eq (sinkpads->data, sinkpad2); + ck_assert_ptr_eq (sinkpads->next->data, sinkpad1); + + /* Get a new pad, which should be the highest pad now */ + sinkpad3 = gst_element_get_request_pad (compositor, "sink_%u"); + sinkpads = GST_ELEMENT (compositor)->sinkpads; + ck_assert_ptr_eq (sinkpads->data, sinkpad2); + ck_assert_ptr_eq (sinkpads->next->data, sinkpad1); + ck_assert_ptr_eq (sinkpads->next->next->data, sinkpad3); + + /* cleanup */ + gst_object_unref (compositor); + gst_object_unref (sinkpad1); + gst_object_unref (sinkpad2); + gst_object_unref (sinkpad3); +} + +GST_END_TEST; + typedef struct { gint buffers_sent; @@ -1628,6 +1672,7 @@ compositor_suite (void) tcase_add_test (tc_chain, test_segment_base_handling); tcase_add_test (tc_chain, test_obscured_skipped); tcase_add_test (tc_chain, test_ignore_eos); + tcase_add_test (tc_chain, test_pad_z_order); tcase_add_test (tc_chain, test_start_time_zero_live_drop_0); tcase_add_test (tc_chain, test_start_time_zero_live_drop_3); tcase_add_test (tc_chain, test_start_time_zero_live_drop_3_unlinked_1); From a238af84a0baa70565945b7e2e809bff4443eb17 Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Mon, 28 Sep 2015 22:20:29 +1000 Subject: [PATCH 222/381] gl: chain up to the parent class for GstElement::set_context https://bugzilla.gnome.org/show_bug.cgi?id=705579 --- ext/gl/gstglbasemixer.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ext/gl/gstglbasemixer.c b/ext/gl/gstglbasemixer.c index 3bad76955d..ef90479412 100644 --- a/ext/gl/gstglbasemixer.c +++ b/ext/gl/gstglbasemixer.c @@ -320,6 +320,8 @@ gst_gl_base_mixer_set_context (GstElement * element, GstContext * context) if (mix->display) gst_gl_display_filter_gl_api (mix->display, mix_class->supported_gl_api); + + GST_ELEMENT_CLASS (parent_class)->set_context (element, context); } static gboolean From b66af464df73bdeee9538bbc23ca1620be8d96d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim-Philipp=20M=C3=BCller?= Date: Tue, 29 Sep 2015 13:31:18 +0100 Subject: [PATCH 223/381] videoaggregator: fix compilation with older glib version Remove weird use of private gtype defines and fix compilation with older glib versions such as 2.36. https://bugzilla.gnome.org/show_bug.cgi?id=755754 --- gst-libs/gst/video/gstvideoaggregator.c | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index 09347c37d8..ea1e8104a7 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -438,25 +438,17 @@ static void gst_videoaggregator_init (GstVideoAggregator * self, GstVideoAggregatorClass * klass); static void gst_videoaggregator_class_init (GstVideoAggregatorClass * klass); static gpointer gst_videoaggregator_parent_class = NULL; -static gint GstVideoAggregator_private_offset; - -_G_DEFINE_TYPE_EXTENDED_CLASS_INIT (GstVideoAggregator, gst_videoaggregator); - -G_GNUC_UNUSED static inline gpointer -gst_videoaggregator_get_instance_private (const GstVideoAggregator * self) -{ - return (G_STRUCT_MEMBER_P (self, GstVideoAggregator_private_offset)); -} GType gst_videoaggregator_get_type (void) { static volatile gsize g_define_type_id_volatile = 0; + if (g_once_init_enter (&g_define_type_id_volatile)) { GType g_define_type_id = g_type_register_static_simple (GST_TYPE_AGGREGATOR, g_intern_static_string ("GstVideoAggregator"), sizeof (GstVideoAggregatorClass), - (GClassInitFunc) gst_videoaggregator_class_intern_init, + (GClassInitFunc) gst_videoaggregator_class_init, sizeof (GstVideoAggregator), (GInstanceInitFunc) gst_videoaggregator_init, (GTypeFlags) G_TYPE_FLAG_ABSTRACT); @@ -2079,6 +2071,8 @@ gst_videoaggregator_class_init (GstVideoAggregatorClass * klass) GST_DEBUG_CATEGORY_INIT (gst_videoaggregator_debug, "videoaggregator", 0, "base video aggregator"); + gst_videoaggregator_parent_class = g_type_class_peek_parent (klass); + g_type_class_add_private (klass, sizeof (GstVideoAggregatorPrivate)); gobject_class->finalize = gst_videoaggregator_finalize; From 1cdc07b3076fe82222f9b48e4e2718490763e3cb Mon Sep 17 00:00:00 2001 From: Nirbheek Chauhan Date: Tue, 6 Oct 2015 21:23:11 +0530 Subject: [PATCH 224/381] glvideomixer: Proxy the ignore-eos videoaggregator property as well Identical to how the z-order property is proxied --- ext/gl/gstglvideomixer.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/ext/gl/gstglvideomixer.c b/ext/gl/gstglvideomixer.c index d17a744945..9376793be2 100644 --- a/ext/gl/gstglvideomixer.c +++ b/ext/gl/gstglvideomixer.c @@ -222,6 +222,7 @@ gst_gl_mixer_control_binding_proxy_new (GstObject * object, #define DEFAULT_PAD_HEIGHT 0 #define DEFAULT_PAD_ALPHA 1.0 #define DEFAULT_PAD_ZORDER 0 +#define DEFAULT_PAD_IGNORE_EOS FALSE enum { @@ -232,6 +233,7 @@ enum PROP_INPUT_HEIGHT, PROP_INPUT_ALPHA, PROP_INPUT_ZORDER, + PROP_INPUT_IGNORE_EOS, }; static void gst_gl_video_mixer_input_get_property (GObject * object, @@ -272,6 +274,11 @@ gst_gl_video_mixer_input_class_init (GstGLVideoMixerInputClass * klass) g_param_spec_uint ("zorder", "Z-Order", "Z Order of the picture", 0, 10000, DEFAULT_PAD_ZORDER, G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); + g_object_class_install_property (gobject_class, PROP_INPUT_IGNORE_EOS, + g_param_spec_boolean ("ignore-eos", "Ignore EOS", "Aggregate the last " + "frame on pads that are EOS till they are released", + DEFAULT_PAD_IGNORE_EOS, + G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); g_object_class_install_property (gobject_class, PROP_INPUT_XPOS, g_param_spec_int ("xpos", "X Position", "X Position of the picture", G_MININT, G_MAXINT, DEFAULT_PAD_XPOS, From 5b07a520387b2696a8c5a7bf5758aef3e62bbf27 Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Thu, 29 Oct 2015 00:44:26 +1100 Subject: [PATCH 225/381] glmemory: add support for rectangle textures Add the various tokens/strings for the differnet texture types (2D, rect, oes) Changes the GLmemory api to include the GstGLTextureTarget in all relevant functions. Update the relevant caps/templates for 2D only textures. --- ext/gl/gstglstereomix.c | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/ext/gl/gstglstereomix.c b/ext/gl/gstglstereomix.c index b880d06357..7b247395d3 100644 --- a/ext/gl/gstglstereomix.c +++ b/ext/gl/gstglstereomix.c @@ -58,9 +58,13 @@ enum static GstStaticPadTemplate src_factory = GST_STATIC_PAD_TEMPLATE ("src", GST_PAD_SRC, GST_PAD_ALWAYS, - GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES - (GST_CAPS_FEATURE_MEMORY_GL_MEMORY, - "RGBA") "; " + GST_STATIC_CAPS ("video/x-raw(" GST_CAPS_FEATURE_MEMORY_GL_MEMORY "), " + "format = (string) RGBA, " + "width = " GST_VIDEO_SIZE_RANGE ", " + "height = " GST_VIDEO_SIZE_RANGE ", " + "framerate = " GST_VIDEO_FPS_RANGE "," + "texture-target = (string) 2D" + "; " GST_VIDEO_CAPS_MAKE_WITH_FEATURES (GST_CAPS_FEATURE_META_GST_VIDEO_GL_TEXTURE_UPLOAD_META, "RGBA") @@ -70,9 +74,13 @@ static GstStaticPadTemplate src_factory = GST_STATIC_PAD_TEMPLATE ("src", static GstStaticPadTemplate sink_factory = GST_STATIC_PAD_TEMPLATE ("sink_%u", GST_PAD_SINK, GST_PAD_REQUEST, - GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES - (GST_CAPS_FEATURE_MEMORY_GL_MEMORY, - "RGBA") "; " + GST_STATIC_CAPS ("video/x-raw(" GST_CAPS_FEATURE_MEMORY_GL_MEMORY "), " + "format = (string) RGBA, " + "width = " GST_VIDEO_SIZE_RANGE ", " + "height = " GST_VIDEO_SIZE_RANGE ", " + "framerate = " GST_VIDEO_FPS_RANGE "," + "texture-target = (string) 2D" + "; " GST_VIDEO_CAPS_MAKE_WITH_FEATURES (GST_CAPS_FEATURE_META_GST_VIDEO_GL_TEXTURE_UPLOAD_META, "RGBA") From 73977da3f9cac44e7a16ae019c21ca8f65143b97 Mon Sep 17 00:00:00 2001 From: Luis de Bethencourt Date: Mon, 2 Nov 2015 16:58:57 +0000 Subject: [PATCH 226/381] videoaggregator: use GST_STIME_ARGS for GstClockTimeDiff No need to manually handle negative values of diff, GST_STIME_ARGS does exactly this. --- gst-libs/gst/video/gstvideoaggregator.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index ea1e8104a7..d3a9ca96dc 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -982,9 +982,9 @@ gst_videoaggregator_update_qos (GstVideoAggregator * vagg, gdouble proportion, gboolean live; GST_DEBUG_OBJECT (vagg, - "Updating QoS: proportion %lf, diff %s%" GST_TIME_FORMAT ", timestamp %" - GST_TIME_FORMAT, proportion, (diff < 0) ? "-" : "", - GST_TIME_ARGS (ABS (diff)), GST_TIME_ARGS (timestamp)); + "Updating QoS: proportion %lf, diff %" GST_STIME_FORMAT ", timestamp %" + GST_TIME_FORMAT, proportion, GST_STIME_ARGS (diff), + GST_TIME_ARGS (timestamp)); live = GST_CLOCK_TIME_IS_VALID (gst_aggregator_get_latency (GST_AGGREGATOR From 2dd8148b300049d50038dcaca682d9928da3201e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim-Philipp=20M=C3=BCller?= Date: Fri, 6 Nov 2015 12:30:12 +0000 Subject: [PATCH 227/381] glmixerbin: fix minor leak Don't leak removed list node. --- ext/gl/gstglmixerbin.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/gl/gstglmixerbin.c b/ext/gl/gstglmixerbin.c index c6488b5cd8..6f83f82a29 100644 --- a/ext/gl/gstglmixerbin.c +++ b/ext/gl/gstglmixerbin.c @@ -458,7 +458,7 @@ gst_gl_mixer_bin_release_pad (GstElement * element, GstPad * pad) struct input_chain *chain = l->data; if (GST_PAD (chain->ghost_pad) == pad) { self->priv->input_chains = - g_list_remove_link (self->priv->input_chains, l); + g_list_delete_link (self->priv->input_chains, l); GST_OBJECT_UNLOCK (element); _free_input_chain (chain); From c5f2c56fe2006a63afcd018502b04a7457b6e9c3 Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Tue, 10 Nov 2015 13:52:30 +1100 Subject: [PATCH 228/381] glviewconvert: remove set_format We need the caps to be able to 1. check the caps features 2. get the requested texture-target on input/output --- ext/gl/gstglstereomix.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/ext/gl/gstglstereomix.c b/ext/gl/gstglstereomix.c index 7b247395d3..c342a23413 100644 --- a/ext/gl/gstglstereomix.c +++ b/ext/gl/gstglstereomix.c @@ -483,6 +483,7 @@ static gboolean _negotiated_caps (GstVideoAggregator * vagg, GstCaps * caps) { GstGLStereoMix *mix = GST_GL_STEREO_MIX (vagg); + GstCaps *in_caps; GST_LOG_OBJECT (mix, "Configured output caps %" GST_PTR_FORMAT, caps); @@ -498,11 +499,16 @@ _negotiated_caps (GstVideoAggregator * vagg, GstCaps * caps) /* We can configure the view_converter now */ gst_gl_view_convert_set_context (mix->viewconvert, GST_GL_BASE_MIXER (mix)->context); - gst_gl_view_convert_set_format (mix->viewconvert, &mix->mix_info, - &mix->out_info); + + in_caps = gst_video_info_to_caps (&mix->mix_info); + gst_caps_set_features (in_caps, 0, + gst_caps_features_from_string (GST_CAPS_FEATURE_MEMORY_GL_MEMORY)); + gst_caps_set_simple (in_caps, "texture-target", G_TYPE_STRING, + GST_GL_TEXTURE_TARGET_2D_STR, NULL); + + gst_gl_view_convert_set_caps (mix->viewconvert, in_caps, caps); return TRUE; - } static gboolean From 353b105816904cc41ddbfdf5f836aa369c2d1530 Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Fri, 11 Dec 2015 15:39:57 +1100 Subject: [PATCH 229/381] gldownload: remove helper api from the library It was never used by anyone and is not needed anymore with the element and GstGLMemory's transparent support for downloading textures. --- ext/gl/gstglstereomix.h | 1 - 1 file changed, 1 deletion(-) diff --git a/ext/gl/gstglstereomix.h b/ext/gl/gstglstereomix.h index debe347c49..0c06d4262a 100644 --- a/ext/gl/gstglstereomix.h +++ b/ext/gl/gstglstereomix.h @@ -50,7 +50,6 @@ struct _GstGLStereoMix GPtrArray *frames; GLuint out_tex_id; - GstGLDownload *download; GstGLViewConvert *viewconvert; GstGLStereoDownmix downmix_mode; From afdc81fe4d61dc407fa5191a8575f61b02606490 Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Tue, 12 Jan 2016 18:21:50 +1100 Subject: [PATCH 230/381] glvideomixer: implement glBlendFunc and glBlendEquation Allows more blending options than just A over B e.g. frame comparisons are now possible. glvideomixer name=m sink_0::zorder=0 sink_1::zorder=1 sink_1::blend-equation-rgb={subtract,reverse-subtract} sink_1::blend-function-src-rgb=src-color sink_1::blend-function-dst-rgb=dst-color ! glimagesinkelement videotestsrc pattern=checkers-4 ! m.sink_0 videotestsrc pattern=checkers-8 ! m.sink_1 --- ext/gl/gstglvideomixer.c | 452 +++++++++++++++++++++++++++++++++++++-- ext/gl/gstglvideomixer.h | 60 ++++++ 2 files changed, 500 insertions(+), 12 deletions(-) diff --git a/ext/gl/gstglvideomixer.c b/ext/gl/gstglvideomixer.c index 9376793be2..f372f0cfb1 100644 --- a/ext/gl/gstglvideomixer.c +++ b/ext/gl/gstglvideomixer.c @@ -71,6 +71,72 @@ gst_gl_video_mixer_background_get_type (void) return mixer_background_type; } +#define GST_TYPE_GL_VIDEO_MIXER_BLEND_EQUATION (gst_gl_video_mixer_blend_equation_get_type()) +static GType +gst_gl_video_mixer_blend_equation_get_type (void) +{ + static GType mixer_blend_equation_type = 0; + + static const GEnumValue mixer_blend_equations[] = { + {GST_GL_VIDEO_MIXER_BLEND_EQUATION_ADD, "Add", "add"}, + {GST_GL_VIDEO_MIXER_BLEND_EQUATION_SUBTRACT, "Subtract", "subtract"}, + {GST_GL_VIDEO_MIXER_BLEND_EQUATION_REVERSE_SUBTRACT, "Reverse Subtract", + "reverse-subtract"}, + {0, NULL, NULL}, + }; + + if (!mixer_blend_equation_type) { + mixer_blend_equation_type = + g_enum_register_static ("GstGLVideoMixerBlendEquation", + mixer_blend_equations); + } + return mixer_blend_equation_type; +} + +#define GST_TYPE_GL_VIDEO_MIXER_BLEND_FUNCTION (gst_gl_video_mixer_blend_function_get_type()) +static GType +gst_gl_video_mixer_blend_function_get_type (void) +{ + static GType mixer_blend_function_type = 0; + + static const GEnumValue mixer_blend_funcs[] = { + {GST_GL_VIDEO_MIXER_BLEND_FUNCTION_ZERO, "Zero", "zero"}, + {GST_GL_VIDEO_MIXER_BLEND_FUNCTION_ONE, "One", "one"}, + {GST_GL_VIDEO_MIXER_BLEND_FUNCTION_SRC_COLOR, "Source Color", "src-color"}, + {GST_GL_VIDEO_MIXER_BLEND_FUNCTION_ONE_MINUS_SRC_COLOR, + "One Minus Source Color", "one-minus-src-color"}, + {GST_GL_VIDEO_MIXER_BLEND_FUNCTION_DST_COLOR, "Destination Color", + "dst-color"}, + {GST_GL_VIDEO_MIXER_BLEND_FUNCTION_ONE_MINUS_DST_COLOR, + "One Minus Destination Color", "one-minus-dst-color"}, + {GST_GL_VIDEO_MIXER_BLEND_FUNCTION_SRC_ALPHA, "Source Alpha", "src-alpha"}, + {GST_GL_VIDEO_MIXER_BLEND_FUNCTION_ONE_MINUS_SRC_ALPHA, + "One Minus Source Alpha", "one-minus-src-alpha"}, + {GST_GL_VIDEO_MIXER_BLEND_FUNCTION_DST_ALPHA, "Destination Alpha", + "dst-alpha"}, + {GST_GL_VIDEO_MIXER_BLEND_FUNCTION_ONE_MINUS_DST_ALPHA, + "One Minus Destination Alpha", "one-minus-dst-alpha"}, + {GST_GL_VIDEO_MIXER_BLEND_FUNCTION_CONSTANT_COLOR, "Constant Color", + "constant-color"}, + {GST_GL_VIDEO_MIXER_BLEND_FUNCTION_ONE_MINUS_CONSTANT_COLOR, + "One Minus Constant Color", "one-minus-contant-color"}, + {GST_GL_VIDEO_MIXER_BLEND_FUNCTION_CONSTANT_ALPHA, "Constant Alpha", + "constant-alpha"}, + {GST_GL_VIDEO_MIXER_BLEND_FUNCTION_ONE_MINUS_CONSTANT_COLOR, + "One Minus Constant Alpha", "one-minus-contant-alpha"}, + {GST_GL_VIDEO_MIXER_BLEND_FUNCTION_SRC_ALPHA_SATURATE, + "Source Alpha Saturate", "src-alpha-saturate"}, + {0, NULL, NULL}, + }; + + if (!mixer_blend_function_type) { + mixer_blend_function_type = + g_enum_register_static ("GstGLVideoMixerBlendFunction", + mixer_blend_funcs); + } + return mixer_blend_function_type; +} + typedef struct _GstGLMixerControlBindingProxy GstGLMixerControlBindingProxy; typedef struct _GstGLMixerControlBindingProxyClass GstGLMixerControlBindingProxyClass; @@ -223,6 +289,12 @@ gst_gl_mixer_control_binding_proxy_new (GstObject * object, #define DEFAULT_PAD_ALPHA 1.0 #define DEFAULT_PAD_ZORDER 0 #define DEFAULT_PAD_IGNORE_EOS FALSE +#define DEFAULT_PAD_BLEND_EQUATION_RGB GST_GL_VIDEO_MIXER_BLEND_EQUATION_ADD +#define DEFAULT_PAD_BLEND_EQUATION_ALPHA GST_GL_VIDEO_MIXER_BLEND_EQUATION_ADD +#define DEFAULT_PAD_BLEND_FUNCTION_SRC_RGB GST_GL_VIDEO_MIXER_BLEND_FUNCTION_SRC_ALPHA +#define DEFAULT_PAD_BLEND_FUNCTION_SRC_ALPHA GST_GL_VIDEO_MIXER_BLEND_FUNCTION_SRC_ALPHA +#define DEFAULT_PAD_BLEND_FUNCTION_DST_RGB GST_GL_VIDEO_MIXER_BLEND_FUNCTION_ONE_MINUS_SRC_ALPHA +#define DEFAULT_PAD_BLEND_FUNCTION_DST_ALPHA GST_GL_VIDEO_MIXER_BLEND_FUNCTION_ONE_MINUS_SRC_ALPHA enum { @@ -232,6 +304,16 @@ enum PROP_INPUT_WIDTH, PROP_INPUT_HEIGHT, PROP_INPUT_ALPHA, + PROP_INPUT_BLEND_EQUATION_RGB, + PROP_INPUT_BLEND_EQUATION_ALPHA, + PROP_INPUT_BLEND_FUNCTION_SRC_RGB, + PROP_INPUT_BLEND_FUNCTION_SRC_ALPHA, + PROP_INPUT_BLEND_FUNCTION_DST_RGB, + PROP_INPUT_BLEND_FUNCTION_DST_ALPHA, + PROP_INPUT_BLEND_FUNCTION_CONSTANT_COLOR_RED, + PROP_INPUT_BLEND_FUNCTION_CONSTANT_COLOR_GREEN, + PROP_INPUT_BLEND_FUNCTION_CONSTANT_COLOR_BLUE, + PROP_INPUT_BLEND_FUNCTION_CONSTANT_COLOR_ALPHA, PROP_INPUT_ZORDER, PROP_INPUT_IGNORE_EOS, }; @@ -299,6 +381,71 @@ gst_gl_video_mixer_input_class_init (GstGLVideoMixerInputClass * klass) g_param_spec_double ("alpha", "Alpha", "Alpha of the picture", 0.0, 1.0, DEFAULT_PAD_ALPHA, G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); + g_object_class_install_property (gobject_class, PROP_INPUT_BLEND_EQUATION_RGB, + g_param_spec_enum ("blend-equation-rgb", "Blend Equation RGB", + "Blend Equation for RGB", + GST_TYPE_GL_VIDEO_MIXER_BLEND_EQUATION, + DEFAULT_PAD_BLEND_EQUATION_RGB, + G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); + g_object_class_install_property (gobject_class, + PROP_INPUT_BLEND_EQUATION_ALPHA, + g_param_spec_enum ("blend-equation-alpha", "Blend Equation Alpha", + "Blend Equation for Alpha", GST_TYPE_GL_VIDEO_MIXER_BLEND_EQUATION, + DEFAULT_PAD_BLEND_EQUATION_ALPHA, + G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); + g_object_class_install_property (gobject_class, + PROP_INPUT_BLEND_FUNCTION_SRC_RGB, + g_param_spec_enum ("blend-function-src-rgb", "Blend Function Source RGB", + "Blend Function for Source RGB", + GST_TYPE_GL_VIDEO_MIXER_BLEND_FUNCTION, + DEFAULT_PAD_BLEND_FUNCTION_SRC_RGB, + G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); + g_object_class_install_property (gobject_class, + PROP_INPUT_BLEND_FUNCTION_SRC_ALPHA, + g_param_spec_enum ("blend-function-src-alpha", + "Blend Function Source Alpha", "Blend Function for Source Alpha", + GST_TYPE_GL_VIDEO_MIXER_BLEND_FUNCTION, + DEFAULT_PAD_BLEND_FUNCTION_SRC_ALPHA, + G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); + g_object_class_install_property (gobject_class, + PROP_INPUT_BLEND_FUNCTION_DST_RGB, + g_param_spec_enum ("blend-function-dst-rgb", + "Blend Function Destination RGB", + "Blend Function for Destination RGB", + GST_TYPE_GL_VIDEO_MIXER_BLEND_FUNCTION, + DEFAULT_PAD_BLEND_FUNCTION_DST_RGB, + G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); + g_object_class_install_property (gobject_class, + PROP_INPUT_BLEND_FUNCTION_DST_ALPHA, + g_param_spec_enum ("blend-function-dst-alpha", + "Blend Function Destination Alpha", + "Blend Function for Destiniation Alpha", + GST_TYPE_GL_VIDEO_MIXER_BLEND_FUNCTION, + DEFAULT_PAD_BLEND_FUNCTION_DST_ALPHA, + G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); + g_object_class_install_property (gobject_class, + PROP_INPUT_BLEND_FUNCTION_CONSTANT_COLOR_RED, + g_param_spec_double ("blend-constant-color-red", + "Blend Constant Color Red", "Blend Constant Color Red", 0.0, 1.0, 0.0, + G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); + g_object_class_install_property (gobject_class, + PROP_INPUT_BLEND_FUNCTION_CONSTANT_COLOR_GREEN, + g_param_spec_double ("blend-constant-color-green", + "Blend Constant Color Green", "Blend Constant Color Green", 0.0, 1.0, + 0.0, + G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); + g_object_class_install_property (gobject_class, + PROP_INPUT_BLEND_FUNCTION_CONSTANT_COLOR_BLUE, + g_param_spec_double ("blend-constant-color-blue", + "Blend Constant Color Green", "Blend Constant Color Green", 0.0, 1.0, + 0.0, + G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); + g_object_class_install_property (gobject_class, + PROP_INPUT_BLEND_FUNCTION_CONSTANT_COLOR_ALPHA, + g_param_spec_double ("blend-constant-color-alpha", + "Blend Constant Color Alpha", "Blend Constant Color Alpha", 0.0, 1.0, + 0.0, + G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); } static void @@ -336,15 +483,25 @@ _create_video_mixer_input (GstGLMixerBin * self, GstPad * mixer_pad) } #define ADD_PROXY_CONTROL_BINDING(prop) \ cb = gst_gl_mixer_control_binding_proxy_new (GST_OBJECT (mixer_pad), \ - G_STRINGIFY (prop), GST_OBJECT (input), G_STRINGIFY (prop)); \ + prop, GST_OBJECT (input), prop); \ gst_object_add_control_binding (GST_OBJECT (mixer_pad), cb) - ADD_PROXY_CONTROL_BINDING (zorder); - ADD_PROXY_CONTROL_BINDING (xpos); - ADD_PROXY_CONTROL_BINDING (ypos); - ADD_PROXY_CONTROL_BINDING (width); - ADD_PROXY_CONTROL_BINDING (height); - ADD_PROXY_CONTROL_BINDING (alpha); + ADD_PROXY_CONTROL_BINDING ("zorder"); + ADD_PROXY_CONTROL_BINDING ("xpos"); + ADD_PROXY_CONTROL_BINDING ("ypos"); + ADD_PROXY_CONTROL_BINDING ("width"); + ADD_PROXY_CONTROL_BINDING ("height"); + ADD_PROXY_CONTROL_BINDING ("alpha"); + ADD_PROXY_CONTROL_BINDING ("blend-equation-rgb"); + ADD_PROXY_CONTROL_BINDING ("blend-equation-alpha"); + ADD_PROXY_CONTROL_BINDING ("blend-function-src-rgb"); + ADD_PROXY_CONTROL_BINDING ("blend-function-src-alpha"); + ADD_PROXY_CONTROL_BINDING ("blend-function-dst-rgb"); + ADD_PROXY_CONTROL_BINDING ("blend-function-dst-alpha"); + ADD_PROXY_CONTROL_BINDING ("blend-constant-color-red"); + ADD_PROXY_CONTROL_BINDING ("blend-constant-color-green"); + ADD_PROXY_CONTROL_BINDING ("blend-constant-color-blue"); + ADD_PROXY_CONTROL_BINDING ("blend-constant-color-alpha"); #undef ADD_PROXY_CONTROL_BINDING @@ -534,6 +691,17 @@ struct _GstGLVideoMixerPad gint width, height; gdouble alpha; + GstGLVideoMixerBlendEquation blend_equation_rgb; + GstGLVideoMixerBlendEquation blend_equation_alpha; + GstGLVideoMixerBlendFunction blend_function_src_rgb; + GstGLVideoMixerBlendFunction blend_function_src_alpha; + GstGLVideoMixerBlendFunction blend_function_dst_rgb; + GstGLVideoMixerBlendFunction blend_function_dst_alpha; + gdouble blend_constant_color_red; + gdouble blend_constant_color_green; + gdouble blend_constant_color_blue; + gdouble blend_constant_color_alpha; + gboolean geometry_change; GLuint vertex_buffer; }; @@ -559,13 +727,29 @@ enum PROP_PAD_YPOS, PROP_PAD_WIDTH, PROP_PAD_HEIGHT, - PROP_PAD_ALPHA + PROP_PAD_ALPHA, + PROP_PAD_BLEND_EQUATION_RGB, + PROP_PAD_BLEND_EQUATION_ALPHA, + PROP_PAD_BLEND_FUNCTION_SRC_RGB, + PROP_PAD_BLEND_FUNCTION_SRC_ALPHA, + PROP_PAD_BLEND_FUNCTION_DST_RGB, + PROP_PAD_BLEND_FUNCTION_DST_ALPHA, + PROP_PAD_BLEND_FUNCTION_CONSTANT_COLOR_RED, + PROP_PAD_BLEND_FUNCTION_CONSTANT_COLOR_GREEN, + PROP_PAD_BLEND_FUNCTION_CONSTANT_COLOR_BLUE, + PROP_PAD_BLEND_FUNCTION_CONSTANT_COLOR_ALPHA, }; static void gst_gl_video_mixer_pad_init (GstGLVideoMixerPad * pad) { - pad->alpha = 1.0; + pad->alpha = DEFAULT_PAD_ALPHA; + pad->blend_equation_rgb = DEFAULT_PAD_BLEND_EQUATION_RGB; + pad->blend_equation_alpha = DEFAULT_PAD_BLEND_EQUATION_ALPHA; + pad->blend_function_src_rgb = DEFAULT_PAD_BLEND_FUNCTION_SRC_RGB; + pad->blend_function_src_alpha = DEFAULT_PAD_BLEND_FUNCTION_SRC_ALPHA; + pad->blend_function_dst_rgb = DEFAULT_PAD_BLEND_FUNCTION_DST_RGB; + pad->blend_function_dst_alpha = DEFAULT_PAD_BLEND_FUNCTION_DST_ALPHA; } static void @@ -596,6 +780,71 @@ gst_gl_video_mixer_pad_class_init (GstGLVideoMixerPadClass * klass) g_param_spec_double ("alpha", "Alpha", "Alpha of the picture", 0.0, 1.0, DEFAULT_PAD_ALPHA, G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); + g_object_class_install_property (gobject_class, PROP_INPUT_BLEND_EQUATION_RGB, + g_param_spec_enum ("blend-equation-rgb", "Blend Equation RGB", + "Blend Equation for RGB", + GST_TYPE_GL_VIDEO_MIXER_BLEND_EQUATION, + DEFAULT_PAD_BLEND_EQUATION_RGB, + G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); + g_object_class_install_property (gobject_class, + PROP_INPUT_BLEND_EQUATION_ALPHA, + g_param_spec_enum ("blend-equation-alpha", "Blend Equation Alpha", + "Blend Equation for Alpha", GST_TYPE_GL_VIDEO_MIXER_BLEND_EQUATION, + DEFAULT_PAD_BLEND_EQUATION_ALPHA, + G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); + g_object_class_install_property (gobject_class, + PROP_INPUT_BLEND_FUNCTION_SRC_RGB, + g_param_spec_enum ("blend-function-src-rgb", "Blend Function Source RGB", + "Blend Function for Source RGB", + GST_TYPE_GL_VIDEO_MIXER_BLEND_FUNCTION, + DEFAULT_PAD_BLEND_FUNCTION_SRC_RGB, + G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); + g_object_class_install_property (gobject_class, + PROP_INPUT_BLEND_FUNCTION_SRC_ALPHA, + g_param_spec_enum ("blend-function-src-alpha", + "Blend Function Source Alpha", "Blend Function for Source Alpha", + GST_TYPE_GL_VIDEO_MIXER_BLEND_FUNCTION, + DEFAULT_PAD_BLEND_FUNCTION_SRC_ALPHA, + G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); + g_object_class_install_property (gobject_class, + PROP_INPUT_BLEND_FUNCTION_DST_RGB, + g_param_spec_enum ("blend-function-dst-rgb", + "Blend Function Destination RGB", + "Blend Function for Destination RGB", + GST_TYPE_GL_VIDEO_MIXER_BLEND_FUNCTION, + DEFAULT_PAD_BLEND_FUNCTION_DST_RGB, + G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); + g_object_class_install_property (gobject_class, + PROP_INPUT_BLEND_FUNCTION_DST_ALPHA, + g_param_spec_enum ("blend-function-dst-alpha", + "Blend Function Destination Alpha", + "Blend Function for Destiniation Alpha", + GST_TYPE_GL_VIDEO_MIXER_BLEND_FUNCTION, + DEFAULT_PAD_BLEND_FUNCTION_DST_ALPHA, + G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); + g_object_class_install_property (gobject_class, + PROP_PAD_BLEND_FUNCTION_CONSTANT_COLOR_RED, + g_param_spec_double ("blend-constant-color-red", + "Blend Constant Color Red", "Blend Constant Color Red", 0.0, 1.0, 0.0, + G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); + g_object_class_install_property (gobject_class, + PROP_PAD_BLEND_FUNCTION_CONSTANT_COLOR_GREEN, + g_param_spec_double ("blend-constant-color-green", + "Blend Constant Color Green", "Blend Constant Color Green", 0.0, 1.0, + 0.0, + G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); + g_object_class_install_property (gobject_class, + PROP_PAD_BLEND_FUNCTION_CONSTANT_COLOR_BLUE, + g_param_spec_double ("blend-constant-color-blue", + "Blend Constant Color Green", "Blend Constant Color Green", 0.0, 1.0, + 0.0, + G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); + g_object_class_install_property (gobject_class, + PROP_PAD_BLEND_FUNCTION_CONSTANT_COLOR_ALPHA, + g_param_spec_double ("blend-constant-color-alpha", + "Blend Constant Color Alpha", "Blend Constant Color Alpha", 0.0, 1.0, + 0.0, + G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); } static void @@ -620,6 +869,36 @@ gst_gl_video_mixer_pad_get_property (GObject * object, guint prop_id, case PROP_PAD_ALPHA: g_value_set_double (value, pad->alpha); break; + case PROP_PAD_BLEND_EQUATION_RGB: + g_value_set_enum (value, pad->blend_equation_rgb); + break; + case PROP_PAD_BLEND_EQUATION_ALPHA: + g_value_set_enum (value, pad->blend_equation_alpha); + break; + case PROP_PAD_BLEND_FUNCTION_SRC_RGB: + g_value_set_enum (value, pad->blend_function_src_rgb); + break; + case PROP_PAD_BLEND_FUNCTION_SRC_ALPHA: + g_value_set_enum (value, pad->blend_function_src_alpha); + break; + case PROP_PAD_BLEND_FUNCTION_DST_RGB: + g_value_set_enum (value, pad->blend_function_dst_rgb); + break; + case PROP_PAD_BLEND_FUNCTION_DST_ALPHA: + g_value_set_enum (value, pad->blend_function_dst_alpha); + break; + case PROP_PAD_BLEND_FUNCTION_CONSTANT_COLOR_RED: + g_value_set_double (value, pad->blend_constant_color_red); + break; + case PROP_PAD_BLEND_FUNCTION_CONSTANT_COLOR_GREEN: + g_value_set_double (value, pad->blend_constant_color_green); + break; + case PROP_PAD_BLEND_FUNCTION_CONSTANT_COLOR_BLUE: + g_value_set_double (value, pad->blend_constant_color_blue); + break; + case PROP_PAD_BLEND_FUNCTION_CONSTANT_COLOR_ALPHA: + g_value_set_double (value, pad->blend_constant_color_alpha); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); break; @@ -653,6 +932,36 @@ gst_gl_video_mixer_pad_set_property (GObject * object, guint prop_id, case PROP_PAD_ALPHA: pad->alpha = g_value_get_double (value); break; + case PROP_PAD_BLEND_EQUATION_RGB: + pad->blend_equation_rgb = g_value_get_enum (value); + break; + case PROP_PAD_BLEND_EQUATION_ALPHA: + pad->blend_equation_alpha = g_value_get_enum (value); + break; + case PROP_PAD_BLEND_FUNCTION_SRC_RGB: + pad->blend_function_src_rgb = g_value_get_enum (value); + break; + case PROP_PAD_BLEND_FUNCTION_SRC_ALPHA: + pad->blend_function_src_alpha = g_value_get_enum (value); + break; + case PROP_PAD_BLEND_FUNCTION_DST_RGB: + pad->blend_function_dst_rgb = g_value_get_enum (value); + break; + case PROP_PAD_BLEND_FUNCTION_DST_ALPHA: + pad->blend_function_dst_alpha = g_value_get_enum (value); + break; + case PROP_PAD_BLEND_FUNCTION_CONSTANT_COLOR_RED: + pad->blend_constant_color_red = g_value_get_double (value); + break; + case PROP_PAD_BLEND_FUNCTION_CONSTANT_COLOR_GREEN: + pad->blend_constant_color_green = g_value_get_double (value); + break; + case PROP_PAD_BLEND_FUNCTION_CONSTANT_COLOR_BLUE: + pad->blend_constant_color_blue = g_value_get_double (value); + break; + case PROP_PAD_BLEND_FUNCTION_CONSTANT_COLOR_ALPHA: + pad->blend_constant_color_alpha = g_value_get_double (value); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); break; @@ -1017,6 +1326,123 @@ _draw_background (GstGLVideoMixer * video_mixer) return TRUE; } +static guint +_blend_equation_to_gl (GstGLVideoMixerBlendEquation equation) +{ + switch (equation) { + case GST_GL_VIDEO_MIXER_BLEND_EQUATION_ADD: + return GL_FUNC_ADD; + case GST_GL_VIDEO_MIXER_BLEND_EQUATION_SUBTRACT: + return GL_FUNC_SUBTRACT; + case GST_GL_VIDEO_MIXER_BLEND_EQUATION_REVERSE_SUBTRACT: + return GL_FUNC_REVERSE_SUBTRACT; + default: + g_assert_not_reached (); + return 0; + } +} + +static guint +_blend_function_to_gl (GstGLVideoMixerBlendFunction equation) +{ + switch (equation) { + case GST_GL_VIDEO_MIXER_BLEND_FUNCTION_ZERO: + return GL_ZERO; + case GST_GL_VIDEO_MIXER_BLEND_FUNCTION_ONE: + return GL_ONE; + case GST_GL_VIDEO_MIXER_BLEND_FUNCTION_SRC_COLOR: + return GL_SRC_COLOR; + case GST_GL_VIDEO_MIXER_BLEND_FUNCTION_ONE_MINUS_SRC_COLOR: + return GL_ONE_MINUS_SRC_COLOR; + case GST_GL_VIDEO_MIXER_BLEND_FUNCTION_DST_COLOR: + return GL_DST_COLOR; + case GST_GL_VIDEO_MIXER_BLEND_FUNCTION_ONE_MINUS_DST_COLOR: + return GL_ONE_MINUS_DST_COLOR; + case GST_GL_VIDEO_MIXER_BLEND_FUNCTION_SRC_ALPHA: + return GL_SRC_ALPHA; + case GST_GL_VIDEO_MIXER_BLEND_FUNCTION_ONE_MINUS_SRC_ALPHA: + return GL_ONE_MINUS_SRC_ALPHA; + case GST_GL_VIDEO_MIXER_BLEND_FUNCTION_DST_ALPHA: + return GL_DST_ALPHA; + case GST_GL_VIDEO_MIXER_BLEND_FUNCTION_ONE_MINUS_DST_ALPHA: + return GL_ONE_MINUS_DST_ALPHA; + case GST_GL_VIDEO_MIXER_BLEND_FUNCTION_CONSTANT_COLOR: + return GL_CONSTANT_COLOR; + case GST_GL_VIDEO_MIXER_BLEND_FUNCTION_ONE_MINUS_CONSTANT_COLOR: + return GL_ONE_MINUS_CONSTANT_COLOR; + case GST_GL_VIDEO_MIXER_BLEND_FUNCTION_CONSTANT_ALPHA: + return GL_CONSTANT_ALPHA; + case GST_GL_VIDEO_MIXER_BLEND_FUNCTION_ONE_MINUS_CONSTANT_ALPHA: + return GL_ONE_MINUS_CONSTANT_ALPHA; + case GST_GL_VIDEO_MIXER_BLEND_FUNCTION_SRC_ALPHA_SATURATE: + return GL_SRC_ALPHA_SATURATE; + default: + g_assert_not_reached (); + return 0; + } +} + +static gboolean +_set_blend_state (GstGLVideoMixer * video_mixer, GstGLVideoMixerPad * mix_pad) +{ + const GstGLFuncs *gl = GST_GL_BASE_MIXER (video_mixer)->context->gl_vtable; + gboolean require_separate = FALSE; + guint gl_func_src_rgb, gl_func_src_alpha, gl_func_dst_rgb, gl_func_dst_alpha; + guint gl_equation_rgb, gl_equation_alpha; + + require_separate = + mix_pad->blend_equation_rgb != mix_pad->blend_equation_alpha + || mix_pad->blend_function_src_rgb != mix_pad->blend_function_src_alpha + || mix_pad->blend_function_dst_rgb != mix_pad->blend_function_dst_alpha; + + if (require_separate && (!gl->BlendFuncSeparate + || !gl->BlendEquationSeparate)) { + GST_ERROR_OBJECT (mix_pad, + "separated blend equations/functions requested however " + "glBlendFuncSeparate or glBlendEquationSeparate not available"); + return FALSE; + } + + if (mix_pad->blend_function_dst_rgb == + GST_GL_VIDEO_MIXER_BLEND_FUNCTION_SRC_ALPHA_SATURATE) { + GST_ERROR_OBJECT (mix_pad, + "Destination RGB blend function cannot be \'SRC_ALPHA_SATURATE\'"); + return FALSE; + } + + if (mix_pad->blend_function_dst_alpha == + GST_GL_VIDEO_MIXER_BLEND_FUNCTION_SRC_ALPHA_SATURATE) { + GST_ERROR_OBJECT (mix_pad, + "Destination alpha blend function cannot be \'SRC_ALPHA_SATURATE\'"); + return FALSE; + } + + gl_equation_rgb = _blend_equation_to_gl (mix_pad->blend_equation_rgb); + gl_equation_alpha = _blend_equation_to_gl (mix_pad->blend_equation_alpha); + + gl_func_src_rgb = _blend_function_to_gl (mix_pad->blend_function_src_rgb); + gl_func_src_alpha = _blend_function_to_gl (mix_pad->blend_function_src_alpha); + gl_func_dst_rgb = _blend_function_to_gl (mix_pad->blend_function_dst_rgb); + gl_func_dst_alpha = _blend_function_to_gl (mix_pad->blend_function_dst_alpha); + + if (gl->BlendEquationSeparate) + gl->BlendEquationSeparate (gl_equation_rgb, gl_equation_alpha); + else + gl->BlendEquation (gl_equation_rgb); + + if (gl->BlendFuncSeparate) + gl->BlendFuncSeparate (gl_func_src_rgb, gl_func_dst_rgb, gl_func_src_alpha, + gl_func_dst_alpha); + else + gl->BlendFunc (gl_func_src_rgb, gl_func_dst_rgb); + + gl->BlendColor (mix_pad->blend_constant_color_red, + mix_pad->blend_constant_color_green, mix_pad->blend_constant_color_blue, + mix_pad->blend_constant_color_alpha); + + return TRUE; +} + /* opengl scene, params: input texture (not the output mixer->texture) */ static void gst_gl_video_mixer_callback (gpointer stuff) @@ -1094,6 +1520,11 @@ gst_gl_video_mixer_callback (gpointer stuff) continue; } + if (!_set_blend_state (video_mixer, pad)) { + GST_FIXME_OBJECT (pad, "skipping due to incorrect blend parameters"); + continue; + } + in_tex = frame->texture; _init_vbo_indices (video_mixer); @@ -1135,9 +1566,6 @@ gst_gl_video_mixer_callback (gpointer stuff) } gl->BindBuffer (GL_ELEMENT_ARRAY_BUFFER, video_mixer->vbo_indices); - gl->BlendFunc (GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA); - gl->BlendEquation (GL_FUNC_ADD); - gl->ActiveTexture (GL_TEXTURE0); gl->BindTexture (GL_TEXTURE_2D, in_tex); gst_gl_shader_set_uniform_1i (video_mixer->shader, "texture", 0); diff --git a/ext/gl/gstglvideomixer.h b/ext/gl/gstglvideomixer.h index 0d0252b56f..86c0506d25 100644 --- a/ext/gl/gstglvideomixer.h +++ b/ext/gl/gstglvideomixer.h @@ -53,6 +53,66 @@ typedef enum } GstGLVideoMixerBackground; +/** + * GstGLVideoMixerBlendEquation: + * @GST_GL_VIDEO_MIXER_BLEND_EQUATION_ADD: Add the two results. + * @GST_GL_VIDEO_MIXER_BLEND_EQUATION_SUBTRACT: Subtract component-wise the destination from the source (S - D). + * @GST_GL_VIDEO_MIXER_BLEND_EQUATION_REVERSE_SUBTRACT: Subtract component-wise the source from the destination (D - S). + * + * The blending equation to use. See the opengl specificition for + * glBlendEquationSeparate + */ +typedef enum +{ + GST_GL_VIDEO_MIXER_BLEND_EQUATION_ADD, + GST_GL_VIDEO_MIXER_BLEND_EQUATION_SUBTRACT, + GST_GL_VIDEO_MIXER_BLEND_EQUATION_REVERSE_SUBTRACT, +} +GstGLVideoMixerBlendEquation; + +/** + * GstGLVideoMixerBlendFunction: + * @GST_GL_VIDEO_MIXER_BLEND_FUNCTION_ZERO: All components are zero + * @GST_GL_VIDEO_MIXER_BLEND_FUNCTION_ONE: All components are one + * @GST_GL_VIDEO_MIXER_BLEND_FUNCTION_SRC_COLOR: Use the source color/alpha + * @GST_GL_VIDEO_MIXER_BLEND_FUNCTION_ONE_MINUS_SRC_COLOR: One minus the source color/alpha + * @GST_GL_VIDEO_MIXER_BLEND_FUNCTION_DST_COLOR: Use the destination color/alpha + * @GST_GL_VIDEO_MIXER_BLEND_FUNCTION_ONE_MINUS_DST_COLOR: One minus the destination color/alpha + * @GST_GL_VIDEO_MIXER_BLEND_FUNCTION_SRC_ALPHA: All components are the source alpha + * @GST_GL_VIDEO_MIXER_BLEND_FUNCTION_ONE_MINUS_SRC_ALPHA: All components are one minus the source alpha + * @GST_GL_VIDEO_MIXER_BLEND_FUNCTION_DST_ALPHA: All components are the destination alpha + * @GST_GL_VIDEO_MIXER_BLEND_FUNCTION_ONE_MINUS_DST_ALPHA: All components are one minus the destination alpha + * @GST_GL_VIDEO_MIXER_BLEND_FUNCTION_CONSTANT_COLOR: Use the constant color/alpha + * @GST_GL_VIDEO_MIXER_BLEND_FUNCTION_ONE_MINUS_CONSTANT_COLOR: Use one minus the constant color/alpha + * @GST_GL_VIDEO_MIXER_BLEND_FUNCTION_CONSTANT_ALPHA: All components are the constant alpha + * @GST_GL_VIDEO_MIXER_BLEND_FUNCTION_ONE_MINUS_CONSTANT_COLOR: All components are one minus the constant alpha + * @GST_GL_VIDEO_MIXER_BLEND_FUNCTION_SRC_ALPHA_SATURATE: All color components + * are the minimum of source alpha and one minus the destination alpha. + * Alpha is equal to one. + * + * The blending function to use. See the opengl specificition for + * glBlendFuncSeparate + */ +typedef enum +{ + GST_GL_VIDEO_MIXER_BLEND_FUNCTION_ZERO, + GST_GL_VIDEO_MIXER_BLEND_FUNCTION_ONE, + GST_GL_VIDEO_MIXER_BLEND_FUNCTION_SRC_COLOR, + GST_GL_VIDEO_MIXER_BLEND_FUNCTION_ONE_MINUS_SRC_COLOR, + GST_GL_VIDEO_MIXER_BLEND_FUNCTION_DST_COLOR, + GST_GL_VIDEO_MIXER_BLEND_FUNCTION_ONE_MINUS_DST_COLOR, + GST_GL_VIDEO_MIXER_BLEND_FUNCTION_SRC_ALPHA, + GST_GL_VIDEO_MIXER_BLEND_FUNCTION_ONE_MINUS_SRC_ALPHA, + GST_GL_VIDEO_MIXER_BLEND_FUNCTION_DST_ALPHA, + GST_GL_VIDEO_MIXER_BLEND_FUNCTION_ONE_MINUS_DST_ALPHA, + GST_GL_VIDEO_MIXER_BLEND_FUNCTION_CONSTANT_COLOR, + GST_GL_VIDEO_MIXER_BLEND_FUNCTION_ONE_MINUS_CONSTANT_COLOR, + GST_GL_VIDEO_MIXER_BLEND_FUNCTION_CONSTANT_ALPHA, + GST_GL_VIDEO_MIXER_BLEND_FUNCTION_ONE_MINUS_CONSTANT_ALPHA, + GST_GL_VIDEO_MIXER_BLEND_FUNCTION_SRC_ALPHA_SATURATE, +} +GstGLVideoMixerBlendFunction; + struct _GstGLVideoMixer { GstGLMixer mixer; From adec074d7e305c2f01a5df94dbeee10d50667350 Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Wed, 27 Jan 2016 19:56:36 +1100 Subject: [PATCH 231/381] gl: move control binding proxy implementation from glvideomixer Other elements may need to use it's functionality --- ext/gl/gstglvideomixer.c | 200 +++++++-------------------------------- 1 file changed, 32 insertions(+), 168 deletions(-) diff --git a/ext/gl/gstglvideomixer.c b/ext/gl/gstglvideomixer.c index f372f0cfb1..9bce937065 100644 --- a/ext/gl/gstglvideomixer.c +++ b/ext/gl/gstglvideomixer.c @@ -137,151 +137,6 @@ gst_gl_video_mixer_blend_function_get_type (void) return mixer_blend_function_type; } -typedef struct _GstGLMixerControlBindingProxy GstGLMixerControlBindingProxy; -typedef struct _GstGLMixerControlBindingProxyClass - GstGLMixerControlBindingProxyClass; - -struct _GstGLMixerControlBindingProxy -{ - GstControlBinding parent; - - GstObject *ref_object; - const gchar *property_name; -}; - -struct _GstGLMixerControlBindingProxyClass -{ - GstControlBindingClass parent_class; -}; - -GType gst_gl_mixer_control_binding_proxy_get_type (void); -#define GST_TYPE_GL_MIXER_CONTROL_BINDING \ - (gst_gl_mixer_control_binding_proxy_get_type()) - -G_DEFINE_TYPE (GstGLMixerControlBindingProxy, - gst_gl_mixer_control_binding_proxy, GST_TYPE_CONTROL_BINDING); - -static void -gst_gl_mixer_control_binding_proxy_init (GstGLMixerControlBindingProxy * self) -{ -} - -static gboolean -gst_gl_mixer_control_binding_proxy_sync_values (GstControlBinding * binding, - GstObject * object, GstClockTime timestamp, GstClockTime last_sync) -{ - GstGLMixerControlBindingProxy *self = (GstGLMixerControlBindingProxy *) - binding; - GstControlBinding *ref_binding; - gboolean ret = TRUE; - - ref_binding = gst_object_get_control_binding (self->ref_object, - self->property_name); - - if (ref_binding) { - ret = gst_control_binding_sync_values (ref_binding, self->ref_object, - timestamp, last_sync); - gst_object_unref (ref_binding); - } - - return ret; -} - -static GValue * -gst_gl_mixer_control_binding_proxy_get_value (GstControlBinding * binding, - GstClockTime timestamp) -{ - GstGLMixerControlBindingProxy *self = (GstGLMixerControlBindingProxy *) - binding; - GstControlBinding *ref_binding; - GValue *ret = NULL; - - ref_binding = gst_object_get_control_binding (self->ref_object, - self->property_name); - - if (ref_binding) { - ret = gst_control_binding_get_value (ref_binding, timestamp); - gst_object_unref (ref_binding); - } - - return ret; -} - -static gboolean -gst_gl_mixer_control_binding_proxy_get_value_array (GstControlBinding * binding, - GstClockTime timestamp, GstClockTime interval, guint n_values, - gpointer values) -{ - GstGLMixerControlBindingProxy *self = (GstGLMixerControlBindingProxy *) - binding; - GstControlBinding *ref_binding; - gboolean ret = FALSE; - - ref_binding = gst_object_get_control_binding (self->ref_object, - self->property_name); - - if (ref_binding) { - ret = gst_control_binding_get_value_array (ref_binding, timestamp, - interval, n_values, values); - gst_object_unref (ref_binding); - } - - return ret; -} - -static gboolean -gst_gl_mixer_control_binding_proxy_get_g_value_array (GstControlBinding * - binding, GstClockTime timestamp, GstClockTime interval, guint n_values, - GValue * values) -{ - GstGLMixerControlBindingProxy *self = (GstGLMixerControlBindingProxy *) - binding; - GstControlBinding *ref_binding; - gboolean ret = FALSE; - - ref_binding = gst_object_get_control_binding (self->ref_object, - self->property_name); - - if (ref_binding) { - ret = gst_control_binding_get_g_value_array (ref_binding, timestamp, - interval, n_values, values); - gst_object_unref (ref_binding); - } - - return ret; -} - - -static void - gst_gl_mixer_control_binding_proxy_class_init - (GstGLMixerControlBindingProxyClass * klass) -{ - GstControlBindingClass *cb_class = GST_CONTROL_BINDING_CLASS (klass); - - cb_class->sync_values = gst_gl_mixer_control_binding_proxy_sync_values; - cb_class->get_value = gst_gl_mixer_control_binding_proxy_get_value; - cb_class->get_value_array = - gst_gl_mixer_control_binding_proxy_get_value_array; - cb_class->get_g_value_array = - gst_gl_mixer_control_binding_proxy_get_g_value_array; -} - -static GstControlBinding * -gst_gl_mixer_control_binding_proxy_new (GstObject * object, - const gchar * property_name, GstObject * ref_object, - const gchar * ref_property_name) -{ - GstGLMixerControlBindingProxy *self = - g_object_new (GST_TYPE_GL_MIXER_CONTROL_BINDING, "object", object, - "name", property_name, NULL); - - self->ref_object = ref_object; - self->property_name = ref_property_name; - - return (GstControlBinding *) self; -} - - #define DEFAULT_PAD_XPOS 0 #define DEFAULT_PAD_YPOS 0 #define DEFAULT_PAD_WIDTH 0 @@ -475,35 +330,44 @@ _create_video_mixer_input (GstGLMixerBin * self, GstPad * mixer_pad) g_object_new (gst_gl_video_mixer_input_get_type (), "name", GST_OBJECT_NAME (mixer_pad), "direction", GST_PAD_DIRECTION (mixer_pad), NULL); - GstControlBinding *cb; if (!gst_ghost_pad_construct (GST_GHOST_PAD (input))) { gst_object_unref (input); return NULL; } -#define ADD_PROXY_CONTROL_BINDING(prop) \ - cb = gst_gl_mixer_control_binding_proxy_new (GST_OBJECT (mixer_pad), \ - prop, GST_OBJECT (input), prop); \ - gst_object_add_control_binding (GST_OBJECT (mixer_pad), cb) - ADD_PROXY_CONTROL_BINDING ("zorder"); - ADD_PROXY_CONTROL_BINDING ("xpos"); - ADD_PROXY_CONTROL_BINDING ("ypos"); - ADD_PROXY_CONTROL_BINDING ("width"); - ADD_PROXY_CONTROL_BINDING ("height"); - ADD_PROXY_CONTROL_BINDING ("alpha"); - ADD_PROXY_CONTROL_BINDING ("blend-equation-rgb"); - ADD_PROXY_CONTROL_BINDING ("blend-equation-alpha"); - ADD_PROXY_CONTROL_BINDING ("blend-function-src-rgb"); - ADD_PROXY_CONTROL_BINDING ("blend-function-src-alpha"); - ADD_PROXY_CONTROL_BINDING ("blend-function-dst-rgb"); - ADD_PROXY_CONTROL_BINDING ("blend-function-dst-alpha"); - ADD_PROXY_CONTROL_BINDING ("blend-constant-color-red"); - ADD_PROXY_CONTROL_BINDING ("blend-constant-color-green"); - ADD_PROXY_CONTROL_BINDING ("blend-constant-color-blue"); - ADD_PROXY_CONTROL_BINDING ("blend-constant-color-alpha"); - -#undef ADD_PROXY_CONTROL_BINDING + gst_gl_object_add_control_binding_proxy (GST_OBJECT (mixer_pad), + GST_OBJECT (input), "zorder"); + gst_gl_object_add_control_binding_proxy (GST_OBJECT (mixer_pad), + GST_OBJECT (input), "xpos"); + gst_gl_object_add_control_binding_proxy (GST_OBJECT (mixer_pad), + GST_OBJECT (input), "ypos"); + gst_gl_object_add_control_binding_proxy (GST_OBJECT (mixer_pad), + GST_OBJECT (input), "width"); + gst_gl_object_add_control_binding_proxy (GST_OBJECT (mixer_pad), + GST_OBJECT (input), "height"); + gst_gl_object_add_control_binding_proxy (GST_OBJECT (mixer_pad), + GST_OBJECT (input), "alpha"); + gst_gl_object_add_control_binding_proxy (GST_OBJECT (mixer_pad), + GST_OBJECT (input), "blend-equation-rgb"); + gst_gl_object_add_control_binding_proxy (GST_OBJECT (mixer_pad), + GST_OBJECT (input), "blend-equation-alpha"); + gst_gl_object_add_control_binding_proxy (GST_OBJECT (mixer_pad), + GST_OBJECT (input), "blend-function-src-rgb"); + gst_gl_object_add_control_binding_proxy (GST_OBJECT (mixer_pad), + GST_OBJECT (input), "blend-function-src-alpha"); + gst_gl_object_add_control_binding_proxy (GST_OBJECT (mixer_pad), + GST_OBJECT (input), "blend-function-dst-rgb"); + gst_gl_object_add_control_binding_proxy (GST_OBJECT (mixer_pad), + GST_OBJECT (input), "blend-function-dst-alpha"); + gst_gl_object_add_control_binding_proxy (GST_OBJECT (mixer_pad), + GST_OBJECT (input), "blend-constant-color-red"); + gst_gl_object_add_control_binding_proxy (GST_OBJECT (mixer_pad), + GST_OBJECT (input), "blend-constant-color-green"); + gst_gl_object_add_control_binding_proxy (GST_OBJECT (mixer_pad), + GST_OBJECT (input), "blend-constant-color-blue"); + gst_gl_object_add_control_binding_proxy (GST_OBJECT (mixer_pad), + GST_OBJECT (input), "blend-constant-color-alpha"); input->mixer_pad = mixer_pad; From f1323fb6df9e1d39707abc609546bd339c112052 Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Wed, 14 Oct 2015 21:13:57 +1100 Subject: [PATCH 232/381] videoaggregator: don't do caps processing that is not overridable Allows the subclass to completely override the chosen src caps. This is needed as videoaggregator generally has no idea exactly what operation is being performed. - Adds a fixate_caps vfunc for fixation - Merges gst_video_aggregator_update_converters() into gst_videoaggregator_update_src_caps() as we need some of its info for proper caps handling. - Pass the downstream caps to the update_caps vfunc https://bugzilla.gnome.org/show_bug.cgi?id=756207 --- ext/gl/gstglmixer.c | 12 +- ext/gl/gstglstereomix.c | 6 +- ext/gl/gstglvideomixer.c | 100 ++++++-- gst-libs/gst/video/gstvideoaggregator.c | 311 +++++++++++------------- gst-libs/gst/video/gstvideoaggregator.h | 11 +- gst/compositor/compositor.c | 90 +++++-- 6 files changed, 295 insertions(+), 235 deletions(-) diff --git a/ext/gl/gstglmixer.c b/ext/gl/gstglmixer.c index 401e4ed6c9..a9daf894ac 100644 --- a/ext/gl/gstglmixer.c +++ b/ext/gl/gstglmixer.c @@ -221,9 +221,17 @@ gst_gl_mixer_pad_sink_acceptcaps (GstPad * pad, GstGLMixer * mix, /* copies the given caps */ static GstCaps * -_update_caps (GstVideoAggregator * vagg, GstCaps * caps) +_update_caps (GstVideoAggregator * vagg, GstCaps * caps, GstCaps * filter) { - return gst_gl_caps_replace_all_caps_features (caps, + GstCaps *tmp; + + if (filter) { + tmp = gst_caps_intersect (caps, filter); + } else { + tmp = caps; + } + + return gst_gl_caps_replace_all_caps_features (tmp, GST_CAPS_FEATURE_MEMORY_GL_MEMORY); } diff --git a/ext/gl/gstglstereomix.c b/ext/gl/gstglstereomix.c index c342a23413..6768ff0106 100644 --- a/ext/gl/gstglstereomix.c +++ b/ext/gl/gstglstereomix.c @@ -33,7 +33,8 @@ GST_DEBUG_CATEGORY (gst_gl_stereo_mix_debug); #define gst_gl_stereo_mix_parent_class parent_class G_DEFINE_TYPE (GstGLStereoMix, gst_gl_stereo_mix, GST_TYPE_GL_MIXER); -static GstCaps *_update_caps (GstVideoAggregator * vagg, GstCaps * caps); +static GstCaps *_update_caps (GstVideoAggregator * vagg, GstCaps * caps, + GstCaps * filter); static gboolean _negotiated_caps (GstVideoAggregator * videoaggregator, GstCaps * caps); gboolean gst_gl_stereo_mix_make_output (GstGLStereoMix * mix); @@ -153,7 +154,6 @@ gst_gl_stereo_mix_class_init (GstGLStereoMixClass * klass) videoaggregator_class->get_output_buffer = gst_gl_stereo_mix_get_output_buffer; videoaggregator_class->find_best_format = gst_gl_stereo_mix_find_best_format; - videoaggregator_class->preserve_update_caps_result = TRUE; base_mix_class->supported_gl_api = GST_GL_API_OPENGL | GST_GL_API_OPENGL3; } @@ -471,7 +471,7 @@ get_converted_caps (GstGLStereoMix * mix, GstCaps * caps) /* Return the possible output caps we decided in find_best_format() */ static GstCaps * -_update_caps (GstVideoAggregator * vagg, GstCaps * caps) +_update_caps (GstVideoAggregator * vagg, GstCaps * caps, GstCaps * filter) { GstGLStereoMix *mix = GST_GL_STEREO_MIX (vagg); diff --git a/ext/gl/gstglvideomixer.c b/ext/gl/gstglvideomixer.c index 9bce937065..a3e22b25d8 100644 --- a/ext/gl/gstglvideomixer.c +++ b/ext/gl/gstglvideomixer.c @@ -460,7 +460,9 @@ static void gst_gl_video_mixer_set_property (GObject * object, guint prop_id, static void gst_gl_video_mixer_get_property (GObject * object, guint prop_id, GValue * value, GParamSpec * pspec); -static GstCaps *_update_caps (GstVideoAggregator * vagg, GstCaps * caps); +static GstCaps *_update_caps (GstVideoAggregator * vagg, GstCaps * caps, + GstCaps * filter); +static GstCaps *_fixate_caps (GstVideoAggregator * vagg, GstCaps * caps); static void gst_gl_video_mixer_reset (GstGLMixer * mixer); static gboolean gst_gl_video_mixer_init_shader (GstGLMixer * mixer, GstCaps * outcaps); @@ -863,6 +865,7 @@ gst_gl_video_mixer_class_init (GstGLVideoMixerClass * klass) gst_gl_video_mixer_process_textures; vagg_class->update_caps = _update_caps; + vagg_class->fixate_caps = _fixate_caps; agg_class->sinkpads_type = GST_TYPE_GL_VIDEO_MIXER_PAD; @@ -912,9 +915,9 @@ gst_gl_video_mixer_get_property (GObject * object, guint prop_id, static void _mixer_pad_get_output_size (GstGLVideoMixer * mix, - GstGLVideoMixerPad * mix_pad, gint * width, gint * height) + GstGLVideoMixerPad * mix_pad, gint out_par_n, gint out_par_d, gint * width, + gint * height) { - GstVideoAggregator *vagg = GST_VIDEO_AGGREGATOR (mix); GstVideoAggregatorPad *vagg_pad = GST_VIDEO_AGGREGATOR_PAD (mix_pad); gint pad_width, pad_height; guint dar_n, dar_d; @@ -937,13 +940,10 @@ _mixer_pad_get_output_size (GstGLVideoMixer * mix, gst_video_calculate_display_ratio (&dar_n, &dar_d, pad_width, pad_height, GST_VIDEO_INFO_PAR_N (&vagg_pad->info), - GST_VIDEO_INFO_PAR_D (&vagg_pad->info), - GST_VIDEO_INFO_PAR_N (&vagg->info), GST_VIDEO_INFO_PAR_D (&vagg->info) - ); + GST_VIDEO_INFO_PAR_D (&vagg_pad->info), out_par_n, out_par_d); GST_LOG_OBJECT (mix_pad, "scaling %ux%u by %u/%u (%u/%u / %u/%u)", pad_width, pad_height, dar_n, dar_d, GST_VIDEO_INFO_PAR_N (&vagg_pad->info), - GST_VIDEO_INFO_PAR_D (&vagg_pad->info), - GST_VIDEO_INFO_PAR_N (&vagg->info), GST_VIDEO_INFO_PAR_D (&vagg->info)); + GST_VIDEO_INFO_PAR_D (&vagg_pad->info), out_par_n, out_par_d); if (pad_height % dar_n == 0) { pad_width = gst_util_uint64_scale_int (pad_height, dar_n, dar_d); @@ -960,17 +960,45 @@ _mixer_pad_get_output_size (GstGLVideoMixer * mix, } static GstCaps * -_update_caps (GstVideoAggregator * vagg, GstCaps * caps) +_update_caps (GstVideoAggregator * vagg, GstCaps * caps, GstCaps * filter) +{ + GstCaps *ret; + + ret = + GST_VIDEO_AGGREGATOR_CLASS (gst_gl_video_mixer_parent_class)->update_caps + (vagg, caps, NULL); + + if (filter) { + GstCaps *tmp = gst_caps_intersect (ret, filter); + gst_caps_unref (ret); + ret = tmp; + } + + return ret; +} + +static GstCaps * +_fixate_caps (GstVideoAggregator * vagg, GstCaps * caps) { GstGLVideoMixer *mix = GST_GL_VIDEO_MIXER (vagg); - GList *l; - gint best_width = -1, best_height = -1; - GstVideoInfo info; + gint best_width = 0, best_height = 0; + gint best_fps_n = 0, best_fps_d = 0; + gint par_n, par_d; + gdouble best_fps = 0.; GstCaps *ret = NULL; - int i; + GstStructure *s; + GList *l; - caps = gst_caps_make_writable (caps); - gst_video_info_from_caps (&info, caps); + ret = gst_caps_make_writable (caps); + + /* we need this to calculate how large to make the output frame */ + s = gst_caps_get_structure (ret, 0); + if (gst_structure_has_field (s, "pixel-aspect-ratio")) { + gst_structure_fixate_field_nearest_fraction (s, "pixel-aspect-ratio", 1, 1); + gst_structure_get_fraction (s, "pixel-aspect-ratio", &par_n, &par_d); + } else { + par_n = par_d = 1; + } GST_OBJECT_LOCK (vagg); for (l = GST_ELEMENT (vagg)->sinkpads; l; l = l->next) { @@ -978,8 +1006,12 @@ _update_caps (GstVideoAggregator * vagg, GstCaps * caps) GstGLVideoMixerPad *mixer_pad = GST_GL_VIDEO_MIXER_PAD (vaggpad); gint this_width, this_height; gint width, height; + gint fps_n, fps_d; + gdouble cur_fps; - _mixer_pad_get_output_size (mix, mixer_pad, &width, &height); + fps_n = GST_VIDEO_INFO_FPS_N (&vaggpad->info); + fps_d = GST_VIDEO_INFO_FPS_D (&vaggpad->info); + _mixer_pad_get_output_size (mix, mixer_pad, par_n, par_d, &width, &height); if (width == 0 || height == 0) continue; @@ -991,20 +1023,34 @@ _update_caps (GstVideoAggregator * vagg, GstCaps * caps) best_width = this_width; if (best_height < this_height) best_height = this_height; + + if (fps_d == 0) + cur_fps = 0.0; + else + gst_util_fraction_to_double (fps_n, fps_d, &cur_fps); + + if (best_fps < cur_fps) { + best_fps = cur_fps; + best_fps_n = fps_n; + best_fps_d = fps_d; + } } GST_OBJECT_UNLOCK (vagg); - ret = - GST_VIDEO_AGGREGATOR_CLASS (gst_gl_video_mixer_parent_class)->update_caps - (vagg, caps); - - for (i = 0; i < gst_caps_get_size (ret); i++) { - GstStructure *s = gst_caps_get_structure (ret, i); - - gst_structure_set (s, "width", G_TYPE_INT, best_width, "height", G_TYPE_INT, - best_height, NULL); + if (best_fps_n <= 0 || best_fps_d <= 0 || best_fps == 0.0) { + best_fps_n = 25; + best_fps_d = 1; + best_fps = 25.0; } + s = gst_caps_get_structure (ret, 0); + gst_structure_fixate_field_nearest_int (s, "width", best_width); + gst_structure_fixate_field_nearest_int (s, "height", best_height); + gst_structure_fixate_field_nearest_fraction (s, "framerate", best_fps_n, + best_fps_d); + gst_structure_fixate_field_nearest_fraction (s, "pixel-aspect-ratio", 1, 1); + ret = gst_caps_fixate (ret); + return ret; } @@ -1397,7 +1443,9 @@ gst_gl_video_mixer_callback (gpointer stuff) gint pad_width, pad_height; gfloat w, h; - _mixer_pad_get_output_size (video_mixer, pad, &pad_width, &pad_height); + _mixer_pad_get_output_size (video_mixer, pad, + GST_VIDEO_INFO_PAR_N (&vagg->info), + GST_VIDEO_INFO_PAR_D (&vagg->info), &pad_width, &pad_height); w = ((gfloat) pad_width / (gfloat) out_width); h = ((gfloat) pad_height / (gfloat) out_height); diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index d3a9ca96dc..2b0c87630e 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -532,86 +532,6 @@ gst_videoaggreagator_find_best_format (GstVideoAggregator * vagg, g_hash_table_unref (formats_table); } -/* WITH GST_VIDEO_AGGREGATOR_LOCK TAKEN - * NOTE: After calling that method you **have to** call - * gst_videoaggregator_update_src_caps (without releasing - * the GST_VIDEO_AGGREGATOR_LOCK in between) - */ -static gboolean -gst_videoaggregator_update_converters (GstVideoAggregator * vagg) -{ - GList *tmp; - GstVideoFormat best_format; - GstVideoInfo best_info; - gboolean at_least_one_alpha = FALSE; - GstCaps *downstream_caps; - GstAggregator *agg = GST_AGGREGATOR (vagg); - - GstVideoAggregatorClass *vagg_class = GST_VIDEO_AGGREGATOR_GET_CLASS (vagg); - GstVideoAggregatorPadClass *vaggpad_class = g_type_class_peek - (GST_AGGREGATOR_GET_CLASS (vagg)->sinkpads_type); - - best_format = GST_VIDEO_FORMAT_UNKNOWN; - gst_video_info_init (&best_info); - - downstream_caps = gst_pad_get_allowed_caps (agg->srcpad); - - if (!downstream_caps || gst_caps_is_empty (downstream_caps)) { - GST_INFO_OBJECT (vagg, "No downstream caps found %" - GST_PTR_FORMAT, downstream_caps); - if (downstream_caps) - gst_caps_unref (downstream_caps); - return FALSE; - } - - - if (vagg_class->find_best_format) { - vagg_class->find_best_format (vagg, downstream_caps, &best_info, - &at_least_one_alpha); - - best_format = GST_VIDEO_INFO_FORMAT (&best_info); - } - - if (best_format == GST_VIDEO_FORMAT_UNKNOWN) { - downstream_caps = gst_caps_fixate (downstream_caps); - gst_video_info_from_caps (&best_info, downstream_caps); - best_format = GST_VIDEO_INFO_FORMAT (&best_info); - } - - gst_caps_unref (downstream_caps); - - if (at_least_one_alpha - && !(best_info.finfo->flags & GST_VIDEO_FORMAT_FLAG_ALPHA)) { - GST_ELEMENT_ERROR (vagg, CORE, NEGOTIATION, - ("At least one of the input pads contains alpha, but downstream can't support alpha."), - ("Either convert your inputs to not contain alpha or add a videoconvert after the aggregator")); - return FALSE; - } - - vagg->info = best_info; - - GST_DEBUG_OBJECT (vagg, - "The output format will now be : %d with chroma : %s", - best_format, gst_video_chroma_to_string (best_info.chroma_site)); - - if (vaggpad_class->set_info) { - GST_OBJECT_LOCK (vagg); - /* Then browse the sinks once more, setting or unsetting conversion if needed */ - for (tmp = GST_ELEMENT (vagg)->sinkpads; tmp; tmp = tmp->next) { - GstVideoAggregatorPad *pad = GST_VIDEO_AGGREGATOR_PAD (tmp->data); - - if (!vaggpad_class->set_info (pad, vagg, &pad->info, &best_info)) { - GST_OBJECT_UNLOCK (vagg); - - return FALSE; - } - } - GST_OBJECT_UNLOCK (vagg); - } - - return TRUE; -} - /* WITH GST_VIDEO_AGGREGATOR_LOCK TAKEN */ static gboolean gst_videoaggregator_src_setcaps (GstVideoAggregator * vagg, GstCaps * caps) @@ -662,25 +582,22 @@ done: return ret; } -/* WITH GST_VIDEO_AGGREGATOR_LOCK TAKEN */ -static gboolean -gst_videoaggregator_update_src_caps (GstVideoAggregator * vagg) +static GstCaps * +gst_videoaggregator_default_fixate_caps (GstVideoAggregator * vagg, + GstCaps * caps) { - GList *l; gint best_width = -1, best_height = -1; - gdouble best_fps = -1, cur_fps; gint best_fps_n = -1, best_fps_d = -1; - gboolean ret = TRUE; - GstElementClass *klass = GST_ELEMENT_GET_CLASS (vagg); - GstVideoAggregatorClass *vagg_klass = (GstVideoAggregatorClass *) klass; - GstAggregator *agg = GST_AGGREGATOR (vagg); + gdouble best_fps = -1.; + GstStructure *s; + GList *l; GST_OBJECT_LOCK (vagg); for (l = GST_ELEMENT (vagg)->sinkpads; l; l = l->next) { GstVideoAggregatorPad *mpad = l->data; - gint this_width, this_height; gint fps_n, fps_d; gint width, height; + gdouble cur_fps; fps_n = GST_VIDEO_INFO_FPS_N (&mpad->info); fps_d = GST_VIDEO_INFO_FPS_D (&mpad->info); @@ -690,13 +607,10 @@ gst_videoaggregator_update_src_caps (GstVideoAggregator * vagg) if (width == 0 || height == 0) continue; - this_width = width; - this_height = height; - - if (best_width < this_width) - best_width = this_width; - if (best_height < this_height) - best_height = this_height; + if (best_width < width) + best_width = width; + if (best_height < height) + best_height = height; if (fps_d == 0) cur_fps = 0.0; @@ -717,88 +631,142 @@ gst_videoaggregator_update_src_caps (GstVideoAggregator * vagg) best_fps = 25.0; } - if (best_width > 0 && best_height > 0 && best_fps > 0) { - GstCaps *caps, *peercaps, *info_caps; - GstStructure *s; - GstVideoInfo info; - int i; + s = gst_caps_get_structure (caps, 0); + gst_structure_fixate_field_nearest_int (s, "width", best_width); + gst_structure_fixate_field_nearest_int (s, "height", best_height); + gst_structure_fixate_field_nearest_fraction (s, "framerate", best_fps_n, + best_fps_d); + if (gst_structure_has_field (s, "pixel-aspect-ratio")) + gst_structure_fixate_field_nearest_fraction (s, "pixel-aspect-ratio", 1, 1); + caps = gst_caps_fixate (caps); - /* Initialize the video info with our target format and - * the best width and height and framerate. Then copy over - * all other fields as we negotiated them before - */ - gst_video_info_set_format (&info, GST_VIDEO_INFO_FORMAT (&vagg->info), - best_width, best_height); - info.fps_n = best_fps_n; - info.fps_d = best_fps_d; - info.chroma_site = vagg->info.chroma_site; - info.par_n = vagg->info.par_n; - info.par_d = vagg->info.par_d; - info.colorimetry = vagg->info.colorimetry; - info.flags = vagg->info.flags; - info.interlace_mode = vagg->info.interlace_mode; + return caps; +} - info_caps = gst_video_info_to_caps (&info); +static GstCaps * +gst_videoaggregator_default_update_caps (GstVideoAggregator * vagg, + GstCaps * caps, GstCaps * filter) +{ + GstCaps *ret; - if (vagg_klass->update_caps) { - if (!(caps = vagg_klass->update_caps (vagg, info_caps))) { - gst_caps_unref (info_caps); - ret = FALSE; - goto done; - } - gst_caps_unref (info_caps); - } else { - caps = info_caps; + if (filter) { + ret = gst_caps_intersect (caps, filter); + } else { + ret = gst_caps_ref (caps); + } + + return ret; +} + +/* WITH GST_VIDEO_AGGREGATOR_LOCK TAKEN */ +static gboolean +gst_videoaggregator_update_src_caps (GstVideoAggregator * vagg) +{ + GstVideoAggregatorClass *vagg_klass = GST_VIDEO_AGGREGATOR_GET_CLASS (vagg); + GstVideoAggregatorPadClass *vaggpad_klass = g_type_class_peek + (GST_AGGREGATOR_GET_CLASS (vagg)->sinkpads_type); + GstAggregator *agg = GST_AGGREGATOR (vagg); + gboolean ret = TRUE, at_least_one_pad_configured = FALSE; + GstVideoFormat best_format; + GstVideoInfo best_info; + gboolean at_least_one_alpha = FALSE; + GstCaps *downstream_caps; + GList *l; + + best_format = GST_VIDEO_FORMAT_UNKNOWN; + gst_video_info_init (&best_info); + + downstream_caps = gst_pad_get_allowed_caps (agg->srcpad); + + if (!downstream_caps || gst_caps_is_empty (downstream_caps)) { + GST_INFO_OBJECT (vagg, "No downstream caps found %" + GST_PTR_FORMAT, downstream_caps); + if (downstream_caps) + gst_caps_unref (downstream_caps); + return FALSE; + } + + if (vagg_klass->find_best_format) { + vagg_klass->find_best_format (vagg, downstream_caps, &best_info, + &at_least_one_alpha); + + best_format = GST_VIDEO_INFO_FORMAT (&best_info); + } + + if (best_format == GST_VIDEO_FORMAT_UNKNOWN) { + GstCaps *tmp = gst_caps_fixate (gst_caps_ref (downstream_caps)); + gst_video_info_from_caps (&best_info, tmp); + best_format = GST_VIDEO_INFO_FORMAT (&best_info); + gst_caps_unref (tmp); + } + + if (at_least_one_alpha + && !(best_info.finfo->flags & GST_VIDEO_FORMAT_FLAG_ALPHA)) { + GST_ELEMENT_ERROR (vagg, CORE, NEGOTIATION, + ("At least one of the input pads contains alpha, but downstream can't support alpha."), + ("Either convert your inputs to not contain alpha or add a videoconvert after the aggregator")); + return FALSE; + } + + GST_DEBUG_OBJECT (vagg, + "The output format will now be : %d with chroma : %s", + best_format, gst_video_chroma_to_string (best_info.chroma_site)); + + GST_OBJECT_LOCK (vagg); + for (l = GST_ELEMENT (vagg)->sinkpads; l; l = l->next) { + GstVideoAggregatorPad *mpad = l->data; + + if (GST_VIDEO_INFO_WIDTH (&mpad->info) == 0 + || GST_VIDEO_INFO_HEIGHT (&mpad->info) == 0) + continue; + + at_least_one_pad_configured = TRUE; + break; + } + GST_OBJECT_UNLOCK (vagg); + + if (at_least_one_pad_configured) { + GstCaps *caps, *peercaps; + + peercaps = gst_pad_peer_query_caps (agg->srcpad, NULL); + + g_assert (vagg_klass->update_caps); + if (!(caps = vagg_klass->update_caps (vagg, downstream_caps, peercaps))) { + GST_WARNING_OBJECT (vagg, "Subclass failed to update provided caps"); + gst_caps_unref (downstream_caps); + if (peercaps) + gst_caps_unref (peercaps); + ret = FALSE; + goto done; } - - /* If the sub-class allows it, allow size/framerate changes */ - if (!vagg_klass->preserve_update_caps_result) { - s = gst_caps_get_structure (caps, 0); - gst_structure_get (s, "width", G_TYPE_INT, &best_width, "height", - G_TYPE_INT, &best_height, NULL); - - for (i = 0; i < gst_caps_get_size (caps); i++) { - s = gst_caps_get_structure (caps, i); - gst_structure_set (s, "width", GST_TYPE_INT_RANGE, 1, G_MAXINT, - "height", GST_TYPE_INT_RANGE, 1, G_MAXINT, "framerate", - GST_TYPE_FRACTION_RANGE, 0, 1, G_MAXINT, 1, NULL); - } - } - - peercaps = gst_pad_peer_query_caps (agg->srcpad, caps); - if (peercaps) { - GstCaps *tmp; - - tmp = gst_caps_intersect (caps, peercaps); - GST_DEBUG_OBJECT (vagg, "intersecting %" GST_PTR_FORMAT - " with peer caps %" GST_PTR_FORMAT " result %" GST_PTR_FORMAT, caps, - peercaps, tmp); - - gst_caps_unref (caps); + gst_caps_unref (downstream_caps); + if (peercaps) gst_caps_unref (peercaps); - caps = tmp; /* pass ownership */ - if (gst_caps_is_empty (caps)) { - GST_DEBUG_OBJECT (vagg, "empty caps"); + + if (!gst_caps_is_fixed (caps)) { + g_assert (vagg_klass->fixate_caps); + + caps = gst_caps_make_writable (caps); + if (!(caps = vagg_klass->fixate_caps (vagg, caps))) { + GST_WARNING_OBJECT (vagg, "Subclass failed to fixate provided caps"); ret = FALSE; - gst_caps_unref (caps); goto done; } + } - caps = gst_caps_truncate (caps); - s = gst_caps_get_structure (caps, 0); - gst_structure_fixate_field_nearest_int (s, "width", best_width); - gst_structure_fixate_field_nearest_int (s, "height", best_height); - gst_structure_fixate_field_nearest_fraction (s, "framerate", best_fps_n, - best_fps_d); - gst_structure_fixate_field_nearest_fraction (s, "pixel-aspect-ratio", 1, - 1); + gst_video_info_from_caps (&vagg->info, caps); - /* fixate the the rest of the fields */ - caps = gst_caps_fixate (caps); + if (vaggpad_klass->set_info) { + /* Then browse the sinks once more, setting or unsetting conversion if needed */ + for (l = GST_ELEMENT (vagg)->sinkpads; l; l = l->next) { + GstVideoAggregatorPad *pad = GST_VIDEO_AGGREGATOR_PAD (l->data); - gst_structure_get_int (s, "width", &info.width); - gst_structure_get_int (s, "height", &info.height); - gst_structure_get_fraction (s, "framerate", &info.fps_n, &info.fps_d); + if (!vaggpad_klass->set_info (pad, vagg, &pad->info, &vagg->info)) { + GST_OBJECT_UNLOCK (vagg); + + return FALSE; + } + } } if (gst_videoaggregator_src_setcaps (vagg, caps)) { @@ -1414,10 +1382,7 @@ gst_videoaggregator_check_reconfigure (GstVideoAggregator * vagg, || gst_pad_check_reconfigure (GST_AGGREGATOR_SRC_PAD (vagg))) { gboolean ret; - ret = gst_videoaggregator_update_converters (vagg); - if (ret) - ret = gst_videoaggregator_update_src_caps (vagg); - + ret = gst_videoaggregator_update_src_caps (vagg); if (!ret) { if (timeout && gst_pad_needs_reconfigure (GST_AGGREGATOR_SRC_PAD (vagg))) { guint64 frame_duration; @@ -2099,6 +2064,8 @@ gst_videoaggregator_class_init (GstVideoAggregatorClass * klass) klass->find_best_format = gst_videoaggreagator_find_best_format; klass->get_output_buffer = gst_videoaggregator_get_output_buffer; + klass->update_caps = gst_videoaggregator_default_update_caps; + klass->fixate_caps = gst_videoaggregator_default_fixate_caps; /* Register the pad class */ g_type_class_ref (GST_TYPE_VIDEO_AGGREGATOR_PAD); diff --git a/gst-libs/gst/video/gstvideoaggregator.h b/gst-libs/gst/video/gstvideoaggregator.h index f95d0d2674..86df0b6294 100644 --- a/gst-libs/gst/video/gstvideoaggregator.h +++ b/gst-libs/gst/video/gstvideoaggregator.h @@ -73,6 +73,9 @@ struct _GstVideoAggregator * @update_caps: Optional. * Lets subclasses update the #GstCaps representing * the src pad caps before usage. Return %NULL to indicate failure. + * @fixate_caps: Fixate and return the src pad caps provided. The function takes + * ownership of @caps and returns a fixated version of + * @caps. @caps is not guaranteed to be writable. * @aggregate_frames: Lets subclasses aggregate frames that are ready. Subclasses * should iterate the GstElement.sinkpads and use the already * mapped #GstVideoFrame from GstVideoAggregatorPad.aggregated_frame @@ -86,9 +89,6 @@ struct _GstVideoAggregator * Notifies subclasses what caps format has been negotiated * @find_best_format: Optional. * Lets subclasses decide of the best common format to use. - * @preserve_update_caps_result: Sub-classes should set this to true if the return result - * of the update_caps() method should not be further modified - * by GstVideoAggregator by removing fields. **/ struct _GstVideoAggregatorClass { @@ -97,6 +97,9 @@ struct _GstVideoAggregatorClass /*< public >*/ GstCaps * (*update_caps) (GstVideoAggregator * videoaggregator, + GstCaps * caps, + GstCaps * filter_caps); + GstCaps * (*fixate_caps) (GstVideoAggregator * videoaggregator, GstCaps * caps); GstFlowReturn (*aggregate_frames) (GstVideoAggregator * videoaggregator, GstBuffer * outbuffer); @@ -109,8 +112,6 @@ struct _GstVideoAggregatorClass GstVideoInfo * best_info, gboolean * at_least_one_alpha); - gboolean preserve_update_caps_result; - GstCaps *sink_non_alpha_caps; /* < private > */ diff --git a/gst/compositor/compositor.c b/gst/compositor/compositor.c index d015d85e10..c5b1a9abf4 100644 --- a/gst/compositor/compositor.c +++ b/gst/compositor/compositor.c @@ -216,9 +216,9 @@ gst_compositor_pad_set_property (GObject * object, guint prop_id, static void _mixer_pad_get_output_size (GstCompositor * comp, - GstCompositorPad * comp_pad, gint * width, gint * height) + GstCompositorPad * comp_pad, gint out_par_n, gint out_par_d, gint * width, + gint * height) { - GstVideoAggregator *vagg = GST_VIDEO_AGGREGATOR (comp); GstVideoAggregatorPad *vagg_pad = GST_VIDEO_AGGREGATOR_PAD (comp_pad); gint pad_width, pad_height; guint dar_n, dar_d; @@ -241,13 +241,10 @@ _mixer_pad_get_output_size (GstCompositor * comp, gst_video_calculate_display_ratio (&dar_n, &dar_d, pad_width, pad_height, GST_VIDEO_INFO_PAR_N (&vagg_pad->info), - GST_VIDEO_INFO_PAR_D (&vagg_pad->info), - GST_VIDEO_INFO_PAR_N (&vagg->info), GST_VIDEO_INFO_PAR_D (&vagg->info) - ); + GST_VIDEO_INFO_PAR_D (&vagg_pad->info), out_par_n, out_par_d); GST_LOG_OBJECT (comp_pad, "scaling %ux%u by %u/%u (%u/%u / %u/%u)", pad_width, pad_height, dar_n, dar_d, GST_VIDEO_INFO_PAR_N (&vagg_pad->info), - GST_VIDEO_INFO_PAR_D (&vagg_pad->info), - GST_VIDEO_INFO_PAR_N (&vagg->info), GST_VIDEO_INFO_PAR_D (&vagg->info)); + GST_VIDEO_INFO_PAR_D (&vagg_pad->info), out_par_n, out_par_d); if (pad_height % dar_n == 0) { pad_width = gst_util_uint64_scale_int (pad_height, dar_n, dar_d); @@ -292,7 +289,8 @@ gst_compositor_pad_set_info (GstVideoAggregatorPad * pad, gst_video_colorimetry_to_string (&(wanted_info->colorimetry)); best_chroma = gst_video_chroma_to_string (wanted_info->chroma_site); - _mixer_pad_get_output_size (comp, cpad, &width, &height); + _mixer_pad_get_output_size (comp, cpad, GST_VIDEO_INFO_PAR_N (&vagg->info), + GST_VIDEO_INFO_PAR_D (&vagg->info), &width, &height); if (GST_VIDEO_INFO_FORMAT (wanted_info) != GST_VIDEO_INFO_FORMAT (current_info) @@ -310,8 +308,8 @@ gst_compositor_pad_set_info (GstVideoAggregatorPad * pad, width, height); tmp_info.chroma_site = wanted_info->chroma_site; tmp_info.colorimetry = wanted_info->colorimetry; - tmp_info.par_n = vagg->info.par_n; - tmp_info.par_d = vagg->info.par_d; + tmp_info.par_n = wanted_info->par_n; + tmp_info.par_d = wanted_info->par_d; tmp_info.fps_n = current_info->fps_n; tmp_info.fps_d = current_info->fps_d; tmp_info.flags = current_info->flags; @@ -403,7 +401,8 @@ gst_compositor_pad_prepare_frame (GstVideoAggregatorPad * pad, * width/height. See ->set_info() * */ - _mixer_pad_get_output_size (comp, cpad, &width, &height); + _mixer_pad_get_output_size (comp, cpad, GST_VIDEO_INFO_PAR_N (&vagg->info), + GST_VIDEO_INFO_PAR_D (&vagg->info), &width, &height); /* The only thing that can change here is the width * and height, otherwise set_info would've been called */ @@ -498,7 +497,8 @@ gst_compositor_pad_prepare_frame (GstVideoAggregatorPad * pad, GstCompositorPad *cpad2 = GST_COMPOSITOR_PAD (pad2); gint pad2_width, pad2_height; - _mixer_pad_get_output_size (comp, cpad2, &pad2_width, &pad2_height); + _mixer_pad_get_output_size (comp, cpad2, GST_VIDEO_INFO_PAR_N (&vagg->info), + GST_VIDEO_INFO_PAR_D (&vagg->info), &pad2_width, &pad2_height); /* We don't need to clamp the coords of the second rectangle */ frame2_rect.x = cpad2->xpos; @@ -883,17 +883,26 @@ set_functions (GstCompositor * self, GstVideoInfo * info) } static GstCaps * -_update_caps (GstVideoAggregator * vagg, GstCaps * caps) +_fixate_caps (GstVideoAggregator * vagg, GstCaps * caps) { GList *l; gint best_width = -1, best_height = -1; - GstVideoInfo info; + gint best_fps_n = -1, best_fps_d = -1; + gint par_n, par_d; + gdouble best_fps = 0.; GstCaps *ret = NULL; + GstStructure *s; - gst_video_info_from_caps (&info, caps); + ret = gst_caps_make_writable (caps); - /* FIXME: this doesn't work for non 1/1 output par's as we don't have that - * information available at this time */ + /* we need this to calculate how large to make the output frame */ + s = gst_caps_get_structure (ret, 0); + if (gst_structure_has_field (s, "pixel-aspect-ratio")) { + gst_structure_fixate_field_nearest_fraction (s, "pixel-aspect-ratio", 1, 1); + gst_structure_get_fraction (s, "pixel-aspect-ratio", &par_n, &par_d); + } else { + par_n = par_d = 1; + } GST_OBJECT_LOCK (vagg); for (l = GST_ELEMENT (vagg)->sinkpads; l; l = l->next) { @@ -901,9 +910,13 @@ _update_caps (GstVideoAggregator * vagg, GstCaps * caps) GstCompositorPad *compositor_pad = GST_COMPOSITOR_PAD (vaggpad); gint this_width, this_height; gint width, height; + gint fps_n, fps_d; + gdouble cur_fps; - _mixer_pad_get_output_size (GST_COMPOSITOR (vagg), compositor_pad, &width, - &height); + fps_n = GST_VIDEO_INFO_FPS_N (&vaggpad->info); + fps_d = GST_VIDEO_INFO_FPS_D (&vaggpad->info); + _mixer_pad_get_output_size (GST_COMPOSITOR (vagg), compositor_pad, par_n, + par_d, &width, &height); if (width == 0 || height == 0) continue; @@ -915,17 +928,40 @@ _update_caps (GstVideoAggregator * vagg, GstCaps * caps) best_width = this_width; if (best_height < this_height) best_height = this_height; + + if (fps_d == 0) + cur_fps = 0.0; + else + gst_util_fraction_to_double (fps_n, fps_d, &cur_fps); + + if (best_fps < cur_fps) { + best_fps = cur_fps; + best_fps_n = fps_n; + best_fps_d = fps_d; + } } GST_OBJECT_UNLOCK (vagg); - if (best_width > 0 && best_height > 0) { - info.width = best_width; - info.height = best_height; - if (set_functions (GST_COMPOSITOR (vagg), &info)) - ret = gst_video_info_to_caps (&info); + if (best_fps_n <= 0 || best_fps_d <= 0 || best_fps == 0.0) { + best_fps_n = 25; + best_fps_d = 1; + best_fps = 25.0; + } - gst_caps_set_simple (ret, "pixel-aspect-ratio", GST_TYPE_FRACTION_RANGE, - 1, G_MAXINT, G_MAXINT, 1, NULL); + gst_structure_fixate_field_nearest_int (s, "width", best_width); + gst_structure_fixate_field_nearest_int (s, "height", best_height); + gst_structure_fixate_field_nearest_fraction (s, "framerate", best_fps_n, + best_fps_d); + ret = gst_caps_fixate (ret); + + if (best_width > 0 && best_height > 0) { + GstVideoInfo v_info; + + gst_video_info_from_caps (&v_info, ret); + if (!set_functions (GST_COMPOSITOR (vagg), &v_info)) { + GST_ERROR_OBJECT (vagg, "Failed to setup vfuncs"); + return NULL; + } } return ret; @@ -1059,7 +1095,7 @@ gst_compositor_class_init (GstCompositorClass * klass) agg_class->sinkpads_type = GST_TYPE_COMPOSITOR_PAD; agg_class->sink_query = _sink_query; - videoaggregator_class->update_caps = _update_caps; + videoaggregator_class->fixate_caps = _fixate_caps; videoaggregator_class->aggregate_frames = gst_compositor_aggregate_frames; g_object_class_install_property (gobject_class, PROP_BACKGROUND, From 5a7f92aa5196d5766d08dbdb3bb4f796b7028227 Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Fri, 29 Jan 2016 14:03:26 +1100 Subject: [PATCH 233/381] glvideomixer: par may not exist in the caps Fixes a critical in the gst-validate tests: gst_structure_fixate_field_nearest_fraction: assertion 'gst_structure_has_field (structure, field_name) --- ext/gl/gstglvideomixer.c | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/ext/gl/gstglvideomixer.c b/ext/gl/gstglvideomixer.c index a3e22b25d8..da295252bc 100644 --- a/ext/gl/gstglvideomixer.c +++ b/ext/gl/gstglvideomixer.c @@ -993,12 +993,11 @@ _fixate_caps (GstVideoAggregator * vagg, GstCaps * caps) /* we need this to calculate how large to make the output frame */ s = gst_caps_get_structure (ret, 0); - if (gst_structure_has_field (s, "pixel-aspect-ratio")) { - gst_structure_fixate_field_nearest_fraction (s, "pixel-aspect-ratio", 1, 1); - gst_structure_get_fraction (s, "pixel-aspect-ratio", &par_n, &par_d); - } else { - par_n = par_d = 1; + if (!gst_structure_has_field (s, "pixel-aspect-ratio")) { + gst_structure_set (s, "pixel-aspect-ratio", GST_TYPE_FRACTION, 1, 1, NULL); } + gst_structure_fixate_field_nearest_fraction (s, "pixel-aspect-ratio", 1, 1); + gst_structure_get_fraction (s, "pixel-aspect-ratio", &par_n, &par_d); GST_OBJECT_LOCK (vagg); for (l = GST_ELEMENT (vagg)->sinkpads; l; l = l->next) { @@ -1048,7 +1047,6 @@ _fixate_caps (GstVideoAggregator * vagg, GstCaps * caps) gst_structure_fixate_field_nearest_int (s, "height", best_height); gst_structure_fixate_field_nearest_fraction (s, "framerate", best_fps_n, best_fps_d); - gst_structure_fixate_field_nearest_fraction (s, "pixel-aspect-ratio", 1, 1); ret = gst_caps_fixate (ret); return ret; From 18a643d36f1181bf614d8e83060d5758b74493ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wang=20Xin-yu=20=28=E7=8E=8B=E6=98=95=E5=AE=87=29?= Date: Thu, 21 Jan 2016 16:10:48 +0800 Subject: [PATCH 234/381] glvideomixer: fix checker vbo leak https://bugzilla.gnome.org/show_bug.cgi?id=760925 --- ext/gl/gstglvideomixer.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ext/gl/gstglvideomixer.c b/ext/gl/gstglvideomixer.c index da295252bc..273d067391 100644 --- a/ext/gl/gstglvideomixer.c +++ b/ext/gl/gstglvideomixer.c @@ -1081,6 +1081,11 @@ _reset_gl (GstGLContext * context, GstGLVideoMixer * video_mixer) video_mixer->vbo_indices = 0; } + if (video_mixer->checker_vbo) { + gl->DeleteBuffers (1, &video_mixer->checker_vbo); + video_mixer->checker_vbo = 0; + } + gst_aggregator_iterate_sinkpads (GST_AGGREGATOR (video_mixer), _reset_pad_gl, NULL); } From a3b2d36abd48024ff8035f1d01074ce313f30f88 Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Wed, 13 Jan 2016 14:41:22 +1100 Subject: [PATCH 235/381] glmixer: don't hold the object lock while calling into GL Doing so can deadlock between the GL thread and the object lock e.g. when performing reconfigure events in glimagesink on a resize event. https://bugzilla.gnome.org/show_bug.cgi?id=760559 --- ext/gl/gstglmixer.c | 115 +++++++++++++++++++++++--------------------- 1 file changed, 60 insertions(+), 55 deletions(-) diff --git a/ext/gl/gstglmixer.c b/ext/gl/gstglmixer.c index a9daf894ac..cfa42c1b54 100644 --- a/ext/gl/gstglmixer.c +++ b/ext/gl/gstglmixer.c @@ -599,16 +599,67 @@ context_error: } } +static gboolean +_upload_frames (GstAggregator * agg, GstAggregatorPad * agg_pad, + gpointer user_data) +{ + GstVideoAggregatorPad *vaggpad = GST_VIDEO_AGGREGATOR_PAD (agg_pad); + GstGLMixerPad *pad = GST_GL_MIXER_PAD (agg_pad); + GstElement *element = GST_ELEMENT (agg); + GstGLMixer *mix = GST_GL_MIXER (agg); + GstGLMixerFrameData *frame; + guint *array_index, i; + + array_index = (guint *) user_data; + + GST_OBJECT_LOCK (agg); + /* make sure the frames array is big enough */ + i = mix->frames->len; + g_ptr_array_set_size (mix->frames, element->numsinkpads); + for (; i < element->numsinkpads; i++) + mix->frames->pdata[i] = g_new0 (GstGLMixerFrameData, 1); + + frame = g_ptr_array_index (mix->frames, *array_index); + frame->pad = pad; + frame->texture = 0; + GST_OBJECT_UNLOCK (agg); + + if (vaggpad->buffer != NULL) { + GstVideoInfo gl_info; + GstVideoFrame gl_frame; + GstGLSyncMeta *sync_meta; + + gst_video_info_set_format (&gl_info, + GST_VIDEO_FORMAT_RGBA, + GST_VIDEO_INFO_WIDTH (&vaggpad->info), + GST_VIDEO_INFO_HEIGHT (&vaggpad->info)); + + sync_meta = gst_buffer_get_gl_sync_meta (vaggpad->buffer); + if (sync_meta) + gst_gl_sync_meta_wait (sync_meta, GST_GL_BASE_MIXER (mix)->context); + + if (!gst_video_frame_map (&gl_frame, &gl_info, vaggpad->buffer, + GST_MAP_READ | GST_MAP_GL)) { + GST_ERROR_OBJECT (agg_pad, "Failed to map input frame"); + return FALSE; + } + + frame->texture = *(guint *) gl_frame.data[0]; + gst_video_frame_unmap (&gl_frame); + } + + (*array_index)++; + + return TRUE; +} + gboolean gst_gl_mixer_process_textures (GstGLMixer * mix, GstBuffer * outbuf) { - guint i; - GList *walk; guint out_tex; gboolean res = TRUE; guint array_index = 0; GstVideoFrame out_frame; - GstElement *element = GST_ELEMENT (mix); GstVideoAggregator *vagg = GST_VIDEO_AGGREGATOR (mix); GstGLMixerClass *mix_class = GST_GL_MIXER_GET_CLASS (mix); GstGLMixerPrivate *priv = mix->priv; @@ -622,47 +673,9 @@ gst_gl_mixer_process_textures (GstGLMixer * mix, GstBuffer * outbuf) out_tex = *(guint *) out_frame.data[0]; - GST_OBJECT_LOCK (mix); - walk = element->sinkpads; - - i = mix->frames->len; - g_ptr_array_set_size (mix->frames, element->numsinkpads); - for (; i < element->numsinkpads; i++) - mix->frames->pdata[i] = g_slice_new0 (GstGLMixerFrameData); - while (walk) { - GstGLMixerPad *pad = GST_GL_MIXER_PAD (walk->data); - GstVideoAggregatorPad *vaggpad = walk->data; - GstGLMixerFrameData *frame; - - frame = g_ptr_array_index (mix->frames, array_index); - frame->pad = pad; - frame->texture = 0; - - walk = g_list_next (walk); - - if (vaggpad->buffer != NULL) { - GstVideoInfo gl_info; - GstVideoFrame gl_frame; - GstGLSyncMeta *sync_meta; - - gst_video_info_set_format (&gl_info, - GST_VIDEO_FORMAT_RGBA, - GST_VIDEO_INFO_WIDTH (&vaggpad->info), - GST_VIDEO_INFO_HEIGHT (&vaggpad->info)); - - sync_meta = gst_buffer_get_gl_sync_meta (vaggpad->buffer); - if (sync_meta) - gst_gl_sync_meta_wait (sync_meta, GST_GL_BASE_MIXER (mix)->context); - - if (gst_video_frame_map (&gl_frame, &gl_info, vaggpad->buffer, - GST_MAP_READ | GST_MAP_GL)) { - frame->texture = *(guint *) gl_frame.data[0]; - gst_video_frame_unmap (&gl_frame); - } - } - - ++array_index; - } + if (!gst_aggregator_iterate_sinkpads (GST_AGGREGATOR (mix), + (GstAggregatorPadForeachFunc) _upload_frames, &array_index)) + return FALSE; g_mutex_lock (&priv->gl_resource_lock); if (!priv->gl_resource_ready) @@ -681,8 +694,6 @@ gst_gl_mixer_process_textures (GstGLMixer * mix, GstBuffer * outbuf) g_mutex_unlock (&priv->gl_resource_lock); out: - GST_OBJECT_UNLOCK (mix); - gst_video_frame_unmap (&out_frame); return res; @@ -701,7 +712,7 @@ gst_gl_mixer_process_buffers (GstGLMixer * mix, GstBuffer * outbuf) i = mix->frames->len; g_ptr_array_set_size (mix->frames, element->numsinkpads); for (; i < element->numsinkpads; i++) - mix->frames->pdata[i] = g_slice_new0 (GstGLMixerFrameData); + mix->frames->pdata[i] = g_new0 (GstGLMixerFrameData, 1); while (walk) { /* We walk with this list because it's ordered */ GstVideoAggregatorPad *vaggpad = walk->data; @@ -761,12 +772,6 @@ gst_gl_mixer_set_property (GObject * object, } } -static void -_free_glmixer_frame_data (GstGLMixerFrameData * frame) -{ - g_slice_free1 (sizeof (GstGLMixerFrameData), frame); -} - static gboolean gst_gl_mixer_start (GstAggregator * agg) { @@ -777,13 +782,13 @@ gst_gl_mixer_start (GstAggregator * agg) GST_OBJECT_LOCK (mix); mix->array_buffers = g_ptr_array_new_full (element->numsinkpads, NULL); mix->frames = g_ptr_array_new_full (element->numsinkpads, - (GDestroyNotify) _free_glmixer_frame_data); + (GDestroyNotify) g_free); g_ptr_array_set_size (mix->array_buffers, element->numsinkpads); g_ptr_array_set_size (mix->frames, element->numsinkpads); for (i = 0; i < element->numsinkpads; i++) - mix->frames->pdata[i] = g_slice_new0 (GstGLMixerFrameData); + mix->frames->pdata[i] = g_new0 (GstGLMixerFrameData, 1); GST_OBJECT_UNLOCK (mix); From 03d1f755faddb9fdb636b87653d6b29a48ba47a2 Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Wed, 17 Feb 2016 01:08:18 +1100 Subject: [PATCH 236/381] glmixer: Remove usage of GstGLMixerFrameData Subclasses can just iterate over the list of pads themselves https://bugzilla.gnome.org/show_bug.cgi?id=760873 --- ext/gl/gstglmixer.c | 75 ++------------------------------ ext/gl/gstglmixer.h | 18 ++------ ext/gl/gstglmosaic.c | 37 +++++++--------- ext/gl/gstglmosaic.h | 1 - ext/gl/gstglstereomix.c | 94 ++++++++++++++-------------------------- ext/gl/gstglstereomix.h | 30 ++++++++----- ext/gl/gstglvideomixer.c | 41 ++++++------------ ext/gl/gstglvideomixer.h | 1 - 8 files changed, 88 insertions(+), 209 deletions(-) diff --git a/ext/gl/gstglmixer.c b/ext/gl/gstglmixer.c index cfa42c1b54..5ca1316ca0 100644 --- a/ext/gl/gstglmixer.c +++ b/ext/gl/gstglmixer.c @@ -416,7 +416,6 @@ static void gst_gl_mixer_init (GstGLMixer * mix) { mix->priv = GST_GL_MIXER_GET_PRIVATE (mix); - mix->array_buffers = 0; mix->fbo = 0; mix->depthbuffer = 0; @@ -605,24 +604,7 @@ _upload_frames (GstAggregator * agg, GstAggregatorPad * agg_pad, { GstVideoAggregatorPad *vaggpad = GST_VIDEO_AGGREGATOR_PAD (agg_pad); GstGLMixerPad *pad = GST_GL_MIXER_PAD (agg_pad); - GstElement *element = GST_ELEMENT (agg); GstGLMixer *mix = GST_GL_MIXER (agg); - GstGLMixerFrameData *frame; - guint *array_index, i; - - array_index = (guint *) user_data; - - GST_OBJECT_LOCK (agg); - /* make sure the frames array is big enough */ - i = mix->frames->len; - g_ptr_array_set_size (mix->frames, element->numsinkpads); - for (; i < element->numsinkpads; i++) - mix->frames->pdata[i] = g_new0 (GstGLMixerFrameData, 1); - - frame = g_ptr_array_index (mix->frames, *array_index); - frame->pad = pad; - frame->texture = 0; - GST_OBJECT_UNLOCK (agg); if (vaggpad->buffer != NULL) { GstVideoInfo gl_info; @@ -644,12 +626,10 @@ _upload_frames (GstAggregator * agg, GstAggregatorPad * agg_pad, return FALSE; } - frame->texture = *(guint *) gl_frame.data[0]; + pad->current_texture = *(guint *) gl_frame.data[0]; gst_video_frame_unmap (&gl_frame); } - (*array_index)++; - return TRUE; } @@ -658,7 +638,6 @@ gst_gl_mixer_process_textures (GstGLMixer * mix, GstBuffer * outbuf) { guint out_tex; gboolean res = TRUE; - guint array_index = 0; GstVideoFrame out_frame; GstVideoAggregator *vagg = GST_VIDEO_AGGREGATOR (mix); GstGLMixerClass *mix_class = GST_GL_MIXER_GET_CLASS (mix); @@ -674,7 +653,7 @@ gst_gl_mixer_process_textures (GstGLMixer * mix, GstBuffer * outbuf) out_tex = *(guint *) out_frame.data[0]; if (!gst_aggregator_iterate_sinkpads (GST_AGGREGATOR (mix), - (GstAggregatorPadForeachFunc) _upload_frames, &array_index)) + (GstAggregatorPadForeachFunc) _upload_frames, NULL)) return FALSE; g_mutex_lock (&priv->gl_resource_lock); @@ -689,7 +668,7 @@ gst_gl_mixer_process_textures (GstGLMixer * mix, GstBuffer * outbuf) goto out; } - mix_class->process_textures (mix, mix->frames, out_tex); + mix_class->process_textures (mix, out_tex); g_mutex_unlock (&priv->gl_resource_lock); @@ -702,31 +681,9 @@ out: static gboolean gst_gl_mixer_process_buffers (GstGLMixer * mix, GstBuffer * outbuf) { - GList *walk; - guint i, array_index = 0; - GstElement *element = GST_ELEMENT (mix); GstGLMixerClass *mix_class = GST_GL_MIXER_GET_CLASS (mix); - GST_OBJECT_LOCK (mix); - walk = GST_ELEMENT (mix)->sinkpads; - i = mix->frames->len; - g_ptr_array_set_size (mix->frames, element->numsinkpads); - for (; i < element->numsinkpads; i++) - mix->frames->pdata[i] = g_new0 (GstGLMixerFrameData, 1); - while (walk) { /* We walk with this list because it's ordered */ - GstVideoAggregatorPad *vaggpad = walk->data; - - walk = g_list_next (walk); - - if (vaggpad->buffer != NULL) { - /* put buffer into array */ - mix->array_buffers->pdata[array_index] = vaggpad->buffer; - } - ++array_index; - } - GST_OBJECT_UNLOCK (mix); - - return mix_class->process_buffers (mix, mix->array_buffers, outbuf); + return mix_class->process_buffers (mix, outbuf); } static GstFlowReturn @@ -775,23 +732,6 @@ gst_gl_mixer_set_property (GObject * object, static gboolean gst_gl_mixer_start (GstAggregator * agg) { - guint i; - GstGLMixer *mix = GST_GL_MIXER (agg); - GstElement *element = GST_ELEMENT (agg); - - GST_OBJECT_LOCK (mix); - mix->array_buffers = g_ptr_array_new_full (element->numsinkpads, NULL); - mix->frames = g_ptr_array_new_full (element->numsinkpads, - (GDestroyNotify) g_free); - - g_ptr_array_set_size (mix->array_buffers, element->numsinkpads); - g_ptr_array_set_size (mix->frames, element->numsinkpads); - - for (i = 0; i < element->numsinkpads; i++) - mix->frames->pdata[i] = g_new0 (GstGLMixerFrameData, 1); - - GST_OBJECT_UNLOCK (mix); - return GST_AGGREGATOR_CLASS (parent_class)->start (agg); } @@ -802,13 +742,6 @@ gst_gl_mixer_stop (GstAggregator * agg) GstGLMixerClass *mixer_class = GST_GL_MIXER_GET_CLASS (mix); GstGLContext *context = GST_GL_BASE_MIXER (mix)->context; - GST_OBJECT_LOCK (agg); - g_ptr_array_free (mix->frames, TRUE); - mix->frames = NULL; - g_ptr_array_free (mix->array_buffers, TRUE); - mix->array_buffers = NULL; - GST_OBJECT_UNLOCK (agg); - if (mixer_class->reset) mixer_class->reset (mix); if (mix->fbo) { diff --git a/ext/gl/gstglmixer.h b/ext/gl/gstglmixer.h index 01eed34f20..81976a748d 100644 --- a/ext/gl/gstglmixer.h +++ b/ext/gl/gstglmixer.h @@ -31,7 +31,6 @@ G_BEGIN_DECLS typedef struct _GstGLMixer GstGLMixer; typedef struct _GstGLMixerClass GstGLMixerClass; typedef struct _GstGLMixerPrivate GstGLMixerPrivate; -typedef struct _GstGLMixerFrameData GstGLMixerFrameData; #define GST_TYPE_GL_MIXER_PAD (gst_gl_mixer_pad_get_type()) #define GST_GL_MIXER_PAD(obj) \ @@ -52,6 +51,8 @@ typedef struct _GstGLMixerPadClass GstGLMixerPadClass; struct _GstGLMixerPad { GstGLBaseMixerPad parent; + + guint current_texture; }; struct _GstGLMixerPadClass @@ -76,18 +77,13 @@ GType gst_gl_mixer_pad_get_type (void); typedef gboolean (*GstGLMixerSetCaps) (GstGLMixer* mixer, GstCaps* outcaps); typedef void (*GstGLMixerReset) (GstGLMixer *mixer); -typedef gboolean (*GstGLMixerProcessFunc) (GstGLMixer *mix, - GPtrArray *buffers, GstBuffer *outbuf); -typedef gboolean (*GstGLMixerProcessTextures) (GstGLMixer *mix, - GPtrArray *frames, guint out_tex); +typedef gboolean (*GstGLMixerProcessFunc) (GstGLMixer *mix, GstBuffer *outbuf); +typedef gboolean (*GstGLMixerProcessTextures) (GstGLMixer *mix, guint out_tex); struct _GstGLMixer { GstGLBaseMixer vaggregator; - GPtrArray *array_buffers; - GPtrArray *frames; - GLuint fbo; GLuint depthbuffer; @@ -106,12 +102,6 @@ struct _GstGLMixerClass GstGLMixerProcessTextures process_textures; }; -struct _GstGLMixerFrameData -{ - GstGLMixerPad *pad; - guint texture; -}; - GType gst_gl_mixer_get_type(void); gboolean gst_gl_mixer_process_textures (GstGLMixer * mix, GstBuffer * outbuf); diff --git a/ext/gl/gstglmosaic.c b/ext/gl/gstglmosaic.c index 0a9834138b..83ac79810e 100644 --- a/ext/gl/gstglmosaic.c +++ b/ext/gl/gstglmosaic.c @@ -70,7 +70,7 @@ static gboolean gst_gl_mosaic_init_shader (GstGLMixer * mixer, GstCaps * outcaps); static gboolean gst_gl_mosaic_process_textures (GstGLMixer * mixer, - GPtrArray * frames, guint out_tex); + guint out_tex); static void gst_gl_mosaic_callback (gpointer stuff); //vertex source @@ -142,7 +142,6 @@ static void gst_gl_mosaic_init (GstGLMosaic * mosaic) { mosaic->shader = NULL; - mosaic->input_frames = NULL; } static void @@ -176,8 +175,6 @@ gst_gl_mosaic_reset (GstGLMixer * mixer) { GstGLMosaic *mosaic = GST_GL_MOSAIC (mixer); - mosaic->input_frames = NULL; - //blocking call, wait the opengl thread has destroyed the shader if (mosaic->shader) gst_gl_context_del_shader (GST_GL_BASE_MIXER (mixer)->context, @@ -196,13 +193,10 @@ gst_gl_mosaic_init_shader (GstGLMixer * mixer, GstCaps * outcaps) } static gboolean -gst_gl_mosaic_process_textures (GstGLMixer * mix, GPtrArray * frames, - guint out_tex) +gst_gl_mosaic_process_textures (GstGLMixer * mix, guint out_tex) { GstGLMosaic *mosaic = GST_GL_MOSAIC (mix); - mosaic->input_frames = frames; - //blocking call, use a FBO gst_gl_context_use_fbo_v2 (GST_GL_BASE_MIXER (mix)->context, GST_VIDEO_INFO_WIDTH (&GST_VIDEO_AGGREGATOR (mix)->info), @@ -219,6 +213,7 @@ gst_gl_mosaic_callback (gpointer stuff) GstGLMosaic *mosaic = GST_GL_MOSAIC (stuff); GstGLMixer *mixer = GST_GL_MIXER (mosaic); GstGLFuncs *gl = GST_GL_BASE_MIXER (mixer)->context->gl_vtable; + GList *walk; static GLfloat xrot = 0; static GLfloat yrot = 0; @@ -255,8 +250,10 @@ gst_gl_mosaic_callback (gpointer stuff) attr_texture_loc = gst_gl_shader_get_attribute_location (mosaic->shader, "a_texCoord"); - while (count < mosaic->input_frames->len && count < 6) { - GstGLMixerFrameData *frame; + GST_OBJECT_LOCK (mosaic); + walk = GST_ELEMENT (mosaic)->sinkpads; + while (walk) { + GstGLMixerPad *pad = walk->data; /* *INDENT-OFF* */ gfloat v_vertices[] = { /* front face */ @@ -294,20 +291,13 @@ gst_gl_mosaic_callback (gpointer stuff) guint in_tex; guint width, height; - frame = g_ptr_array_index (mosaic->input_frames, count); - if (!frame) { - GST_DEBUG ("skipping texture, null frame"); - count++; - continue; - } - in_tex = frame->texture; - width = GST_VIDEO_INFO_WIDTH (&GST_VIDEO_AGGREGATOR_PAD (frame->pad)->info); - height = - GST_VIDEO_INFO_HEIGHT (&GST_VIDEO_AGGREGATOR_PAD (frame->pad)->info); + in_tex = pad->current_texture; + width = GST_VIDEO_INFO_WIDTH (&GST_VIDEO_AGGREGATOR_PAD (pad)->info); + height = GST_VIDEO_INFO_HEIGHT (&GST_VIDEO_AGGREGATOR_PAD (pad)->info); if (!in_tex || width <= 0 || height <= 0) { - GST_DEBUG ("skipping texture:%u frame:%p width:%u height %u", - in_tex, frame, width, height); + GST_DEBUG ("skipping texture:%u pad:%p width:%u height %u", + in_tex, pad, width, height); count++; continue; } @@ -335,7 +325,10 @@ gst_gl_mosaic_callback (gpointer stuff) gl->DrawElements (GL_TRIANGLES, 6, GL_UNSIGNED_SHORT, indices); ++count; + + walk = g_list_next (walk); } + GST_OBJECT_UNLOCK (mosaic); gl->DisableVertexAttribArray (attr_position_loc); gl->DisableVertexAttribArray (attr_texture_loc); diff --git a/ext/gl/gstglmosaic.h b/ext/gl/gstglmosaic.h index 49c99e7754..e0b340cc28 100644 --- a/ext/gl/gstglmosaic.h +++ b/ext/gl/gstglmosaic.h @@ -40,7 +40,6 @@ struct _GstGLMosaic GstGLMixer mixer; GstGLShader *shader; - GPtrArray *input_frames; }; struct _GstGLMosaicClass diff --git a/ext/gl/gstglstereomix.c b/ext/gl/gstglstereomix.c index 6768ff0106..fee1080c26 100644 --- a/ext/gl/gstglstereomix.c +++ b/ext/gl/gstglstereomix.c @@ -30,6 +30,18 @@ #define GST_CAT_DEFAULT gst_gl_stereo_mix_debug GST_DEBUG_CATEGORY (gst_gl_stereo_mix_debug); +G_DEFINE_TYPE (GstGLStereoMixPad, gst_gl_stereo_mix_pad, GST_TYPE_GL_MIXER_PAD); + +static void +gst_gl_stereo_mix_pad_class_init (GstGLStereoMixPadClass * klass) +{ +} + +static void +gst_gl_stereo_mix_pad_init (GstGLStereoMixPad * pad) +{ +} + #define gst_gl_stereo_mix_parent_class parent_class G_DEFINE_TYPE (GstGLStereoMix, gst_gl_stereo_mix, GST_TYPE_GL_MIXER); @@ -38,8 +50,7 @@ static GstCaps *_update_caps (GstVideoAggregator * vagg, GstCaps * caps, static gboolean _negotiated_caps (GstVideoAggregator * videoaggregator, GstCaps * caps); gboolean gst_gl_stereo_mix_make_output (GstGLStereoMix * mix); -static gboolean gst_gl_stereo_mix_process_frames (GstGLStereoMix * mixer, - GPtrArray * in_frames); +static gboolean gst_gl_stereo_mix_process_frames (GstGLStereoMix * mixer); #define DEFAULT_DOWNMIX GST_GL_STEREO_DOWNMIX_ANAGLYPH_GREEN_MAGENTA_DUBOIS @@ -144,6 +155,7 @@ gst_gl_stereo_mix_class_init (GstGLStereoMixClass * klass) gst_element_class_add_pad_template (element_class, gst_static_pad_template_get (&sink_factory)); + agg_class->sinkpads_type = GST_TYPE_GL_STEREO_MIX_PAD; agg_class->stop = gst_gl_stereo_mix_stop; agg_class->start = gst_gl_stereo_mix_start; agg_class->src_query = gst_gl_stereo_mix_src_query; @@ -250,10 +262,8 @@ gst_gl_stereo_mix_get_output_buffer (GstVideoAggregator * videoaggregator, gboolean gst_gl_stereo_mix_make_output (GstGLStereoMix * mix) { - guint i; GList *walk; gboolean res = FALSE; - guint array_index = 0; GstElement *element = GST_ELEMENT (mix); gboolean missing_buffer = FALSE; @@ -261,33 +271,23 @@ gst_gl_stereo_mix_make_output (GstGLStereoMix * mix) GST_OBJECT_LOCK (mix); walk = element->sinkpads; - - i = mix->frames->len; - g_ptr_array_set_size (mix->frames, element->numsinkpads); - for (; i < element->numsinkpads; i++) - mix->frames->pdata[i] = g_slice_new0 (GstGLStereoMixFrameData); while (walk) { - GstGLMixerPad *pad = GST_GL_MIXER_PAD (walk->data); GstVideoAggregatorPad *vaggpad = walk->data; - GstGLStereoMixFrameData *frame; + GstGLStereoMixPad *pad = walk->data; GST_LOG_OBJECT (mix, "Checking pad %" GST_PTR_FORMAT, vaggpad); - frame = g_ptr_array_index (mix->frames, array_index); - frame->base.pad = pad; - frame->buf = NULL; - - walk = g_list_next (walk); - if (vaggpad->buffer != NULL) { - frame->buf = vaggpad->buffer; + pad->current_buffer = vaggpad->buffer; - GST_DEBUG_OBJECT (pad, "Got buffer %" GST_PTR_FORMAT, frame->buf); + GST_DEBUG_OBJECT (pad, "Got buffer %" GST_PTR_FORMAT, + pad->current_buffer); } else { GST_LOG_OBJECT (mix, "No buffer on pad %" GST_PTR_FORMAT, vaggpad); + pad->current_buffer = NULL; missing_buffer = TRUE; } - ++array_index; + walk = g_list_next (walk); } if (missing_buffer) { /* We're still waiting for a buffer to turn up on at least one input */ @@ -297,7 +297,7 @@ gst_gl_stereo_mix_make_output (GstGLStereoMix * mix) } /* Copy GL memory from each input frame to the output */ - if (!gst_gl_stereo_mix_process_frames (mix, mix->frames)) { + if (!gst_gl_stereo_mix_process_frames (mix)) { GST_LOG_OBJECT (mix, "Failed to process frames to output"); goto out; } @@ -371,41 +371,18 @@ gst_gl_stereo_mix_set_property (GObject * object, } } -static void -_free_glmixer_frame_data (GstGLStereoMixFrameData * frame) -{ - if (frame == NULL) - return; - if (frame->buf) - gst_buffer_unref (frame->buf); - g_slice_free1 (sizeof (GstGLStereoMixFrameData), frame); -} - static gboolean gst_gl_stereo_mix_start (GstAggregator * agg) { - guint i; GstGLStereoMix *mix = GST_GL_STEREO_MIX (agg); - GstElement *element = GST_ELEMENT (agg); if (!GST_AGGREGATOR_CLASS (parent_class)->start (agg)) return FALSE; GST_OBJECT_LOCK (mix); - mix->array_buffers = g_ptr_array_new_full (element->numsinkpads, - (GDestroyNotify) _free_glmixer_frame_data); - mix->frames = g_ptr_array_new_full (element->numsinkpads, NULL); - - g_ptr_array_set_size (mix->array_buffers, element->numsinkpads); - g_ptr_array_set_size (mix->frames, element->numsinkpads); - - for (i = 0; i < element->numsinkpads; i++) - mix->frames->pdata[i] = g_slice_new0 (GstGLStereoMixFrameData); - mix->viewconvert = gst_gl_view_convert_new (); g_object_set (G_OBJECT (mix->viewconvert), "downmix-mode", mix->downmix_mode, NULL); - GST_OBJECT_UNLOCK (mix); return TRUE; @@ -419,13 +396,6 @@ gst_gl_stereo_mix_stop (GstAggregator * agg) if (!GST_AGGREGATOR_CLASS (parent_class)->stop (agg)) return FALSE; - GST_OBJECT_LOCK (agg); - g_ptr_array_free (mix->frames, TRUE); - mix->frames = NULL; - g_ptr_array_free (mix->array_buffers, TRUE); - mix->array_buffers = NULL; - GST_OBJECT_UNLOCK (agg); - if (mix->viewconvert) { gst_object_unref (mix->viewconvert); mix->viewconvert = NULL; @@ -511,34 +481,34 @@ _negotiated_caps (GstVideoAggregator * vagg, GstCaps * caps) return TRUE; } +/* called with the object lock held */ static gboolean -gst_gl_stereo_mix_process_frames (GstGLStereoMix * mixer, GPtrArray * frames) +gst_gl_stereo_mix_process_frames (GstGLStereoMix * mixer) { GstVideoAggregator *vagg = GST_VIDEO_AGGREGATOR (mixer); GstBuffer *converted_buffer, *inbuf; GstVideoInfo *out_info = &vagg->info; - gint count = 0; #ifndef G_DISABLE_ASSERT gint n; #endif gint v, views; gint valid_views = 0; + GList *walk; inbuf = gst_buffer_new (); - while (count < frames->len) { - GstGLStereoMixFrameData *frame; + walk = GST_ELEMENT (mixer)->sinkpads; + while (walk) { + GstGLStereoMixPad *pad = walk->data; GstMemory *in_mem; - frame = g_ptr_array_index (frames, count); - GST_LOG_OBJECT (mixer, "Handling frame %d", count); + GST_LOG_OBJECT (mixer, "Handling frame %d", valid_views); - if (!frame) { + if (!pad || !pad->current_buffer) { GST_DEBUG ("skipping texture, null frame"); - count++; continue; } - in_mem = gst_buffer_get_memory (frame->buf, 0); + in_mem = gst_buffer_get_memory (pad->current_buffer, 0); GST_LOG_OBJECT (mixer, "Appending memory %" GST_PTR_FORMAT " to intermediate buffer", in_mem); @@ -551,10 +521,10 @@ gst_gl_stereo_mix_process_frames (GstGLStereoMix * mixer, GPtrArray * frames) */ gst_buffer_append_memory (inbuf, in_mem); /* Use parent buffer meta to keep input buffer alive */ - gst_buffer_add_parent_buffer_meta (inbuf, frame->buf); + gst_buffer_add_parent_buffer_meta (inbuf, pad->current_buffer); - count++; valid_views++; + walk = g_list_next (walk); } if (mixer->mix_info.views != valid_views) { diff --git a/ext/gl/gstglstereomix.h b/ext/gl/gstglstereomix.h index 0c06d4262a..b0f1bd232c 100644 --- a/ext/gl/gstglstereomix.h +++ b/ext/gl/gstglstereomix.h @@ -40,15 +40,30 @@ G_BEGIN_DECLS typedef struct _GstGLStereoMix GstGLStereoMix; typedef struct _GstGLStereoMixClass GstGLStereoMixClass; -typedef struct _GstGLStereoMixFrameData GstGLStereoMixFrameData; +typedef struct _GstGLStereoMixPad GstGLStereoMixPad; +typedef struct _GstGLStereoMixPadClass GstGLStereoMixPadClass; + +struct _GstGLStereoMixPad +{ + GstGLMixerPad mixer_pad; + + gboolean mapped; + GstBuffer *current_buffer; +}; + +struct _GstGLStereoMixPadClass +{ + GstGLMixerPadClass mixer_pad_class; +}; + +#define GST_TYPE_GL_STEREO_MIX_PAD (gst_gl_stereo_mix_pad_get_type ()) +GType gst_gl_stereo_mix_pad_get_type (void); + struct _GstGLStereoMix { GstGLMixer mixer; - GPtrArray *array_buffers; - GPtrArray *frames; - GLuint out_tex_id; GstGLViewConvert *viewconvert; @@ -69,13 +84,6 @@ struct _GstGLStereoMixClass GstGLMixerClass mixer_class; }; -struct _GstGLStereoMixFrameData -{ - GstGLMixerFrameData base; - gboolean mapped; - GstBuffer *buf; -}; - GType gst_gl_stereo_mix_get_type(void); G_END_DECLS diff --git a/ext/gl/gstglvideomixer.c b/ext/gl/gstglvideomixer.c index 273d067391..e0fd427756 100644 --- a/ext/gl/gstglvideomixer.c +++ b/ext/gl/gstglvideomixer.c @@ -468,7 +468,7 @@ static gboolean gst_gl_video_mixer_init_shader (GstGLMixer * mixer, GstCaps * outcaps); static gboolean gst_gl_video_mixer_process_textures (GstGLMixer * mixer, - GPtrArray * in_frames, guint out_tex); + guint out_tex); static void gst_gl_video_mixer_callback (gpointer stuff); /* *INDENT-OFF* */ @@ -878,7 +878,6 @@ gst_gl_video_mixer_init (GstGLVideoMixer * video_mixer) { video_mixer->background = DEFAULT_BACKGROUND; video_mixer->shader = NULL; - video_mixer->input_frames = NULL; } static void @@ -1096,8 +1095,6 @@ gst_gl_video_mixer_reset (GstGLMixer * mixer) GstGLVideoMixer *video_mixer = GST_GL_VIDEO_MIXER (mixer); GstGLContext *context = GST_GL_BASE_MIXER (mixer)->context; - video_mixer->input_frames = NULL; - GST_DEBUG_OBJECT (mixer, "context:%p", context); if (video_mixer->shader) @@ -1127,13 +1124,10 @@ gst_gl_video_mixer_init_shader (GstGLMixer * mixer, GstCaps * outcaps) } static gboolean -gst_gl_video_mixer_process_textures (GstGLMixer * mix, GPtrArray * frames, - guint out_tex) +gst_gl_video_mixer_process_textures (GstGLMixer * mix, guint out_tex) { GstGLVideoMixer *video_mixer = GST_GL_VIDEO_MIXER (mix); - video_mixer->input_frames = frames; - gst_gl_context_use_fbo_v2 (GST_GL_BASE_MIXER (mix)->context, GST_VIDEO_INFO_WIDTH (&GST_VIDEO_AGGREGATOR (mix)->info), GST_VIDEO_INFO_HEIGHT (&GST_VIDEO_AGGREGATOR (mix)->info), @@ -1364,12 +1358,10 @@ gst_gl_video_mixer_callback (gpointer stuff) GstVideoAggregator *vagg = GST_VIDEO_AGGREGATOR (stuff); GstGLMixer *mixer = GST_GL_MIXER (video_mixer); GstGLFuncs *gl = GST_GL_BASE_MIXER (mixer)->context->gl_vtable; - GLint attr_position_loc = 0; GLint attr_texture_loc = 0; guint out_width, out_height; - - guint count = 0; + GList *walk; out_width = GST_VIDEO_INFO_WIDTH (&vagg->info); out_height = GST_VIDEO_INFO_HEIGHT (&vagg->info); @@ -1398,9 +1390,11 @@ gst_gl_video_mixer_callback (gpointer stuff) gl->Enable (GL_BLEND); - while (count < video_mixer->input_frames->len) { - GstGLMixerFrameData *frame; - GstGLVideoMixerPad *pad; + GST_OBJECT_LOCK (video_mixer); + walk = GST_ELEMENT (video_mixer)->sinkpads; + while (walk) { + GstGLMixerPad *mix_pad = walk->data; + GstGLVideoMixerPad *pad = walk->data; GstVideoInfo *v_info; guint in_tex; guint in_width, in_height; @@ -1414,22 +1408,14 @@ gst_gl_video_mixer_callback (gpointer stuff) }; /* *INDENT-ON* */ - frame = g_ptr_array_index (video_mixer->input_frames, count); - if (!frame) { - GST_DEBUG ("skipping texture, null frame"); - count++; - continue; - } - pad = (GstGLVideoMixerPad *) frame->pad; v_info = &GST_VIDEO_AGGREGATOR_PAD (pad)->info; in_width = GST_VIDEO_INFO_WIDTH (v_info); in_height = GST_VIDEO_INFO_HEIGHT (v_info); - if (!frame->texture || in_width <= 0 || in_height <= 0 + if (!mix_pad->current_texture || in_width <= 0 || in_height <= 0 || pad->alpha == 0.0f) { - GST_DEBUG ("skipping texture:%u frame:%p width:%u height:%u alpha:%f", - frame->texture, frame, in_width, in_height, pad->alpha); - count++; + GST_DEBUG ("skipping texture:%u pad:%p width:%u height:%u alpha:%f", + mix_pad->current_texture, pad, in_width, in_height, pad->alpha); continue; } @@ -1438,7 +1424,7 @@ gst_gl_video_mixer_callback (gpointer stuff) continue; } - in_tex = frame->texture; + in_tex = mix_pad->current_texture; _init_vbo_indices (video_mixer); @@ -1497,8 +1483,9 @@ gst_gl_video_mixer_callback (gpointer stuff) gl->DrawElements (GL_TRIANGLES, 6, GL_UNSIGNED_SHORT, 0); - ++count; + walk = g_list_next (walk); } + GST_OBJECT_UNLOCK (video_mixer); gl->DisableVertexAttribArray (attr_position_loc); gl->DisableVertexAttribArray (attr_texture_loc); diff --git a/ext/gl/gstglvideomixer.h b/ext/gl/gstglvideomixer.h index 86c0506d25..a0776fd9fc 100644 --- a/ext/gl/gstglvideomixer.h +++ b/ext/gl/gstglvideomixer.h @@ -121,7 +121,6 @@ struct _GstGLVideoMixer GstGLShader *shader; GstGLShader *checker; - GPtrArray *input_frames; GLuint vao; GLuint vbo_indices; From 282c7eca0652249d438ffa2758aba1b1a5ad038a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wang=20Xin-yu=20=28=E7=8E=8B=E6=98=95=E5=AE=87=29?= Date: Thu, 21 Jan 2016 10:40:36 +0800 Subject: [PATCH 237/381] glvideomixer: don't leak pad's vertex buffer on release_pad https://bugzilla.gnome.org/show_bug.cgi?id=760873 --- ext/gl/gstglvideomixer.c | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/ext/gl/gstglvideomixer.c b/ext/gl/gstglvideomixer.c index e0fd427756..e48151bd04 100644 --- a/ext/gl/gstglvideomixer.c +++ b/ext/gl/gstglvideomixer.c @@ -836,6 +836,26 @@ gst_gl_video_mixer_pad_set_property (GObject * object, guint prop_id, gst_object_unref (mix); } +static void +_del_buffer (GstGLContext * context, GLuint * pBuffer) +{ + context->gl_vtable->DeleteBuffers (1, pBuffer); +} + +static void +gst_gl_video_mixer_release_pad (GstElement * element, GstPad * p) +{ + GstGLVideoMixerPad *pad = GST_GL_VIDEO_MIXER_PAD (p); + if (pad->vertex_buffer) { + GstGLBaseMixer *mix = GST_GL_BASE_MIXER (element); + gst_gl_context_thread_add (mix->context, (GstGLContextThreadFunc) + _del_buffer, &pad->vertex_buffer); + pad->vertex_buffer = 0; + } + GST_ELEMENT_CLASS (g_type_class_peek_parent (G_OBJECT_GET_CLASS (element))) + ->release_pad (element, p); +} + static void gst_gl_video_mixer_class_init (GstGLVideoMixerClass * klass) { @@ -846,6 +866,7 @@ gst_gl_video_mixer_class_init (GstGLVideoMixerClass * klass) gobject_class = (GObjectClass *) klass; element_class = GST_ELEMENT_CLASS (klass); + element_class->release_pad = gst_gl_video_mixer_release_pad; gobject_class->set_property = gst_gl_video_mixer_set_property; gobject_class->get_property = gst_gl_video_mixer_get_property; From 0d50d9227915f21451c31b7f6bf2cb0895e09514 Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Mon, 22 Feb 2016 20:49:52 +1100 Subject: [PATCH 238/381] gl: error out if the configured GL API is unsupported by our element https://bugzilla.gnome.org/show_bug.cgi?id=759801 --- ext/gl/gstglbasemixer.c | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/ext/gl/gstglbasemixer.c b/ext/gl/gstglbasemixer.c index ef90479412..d207883776 100644 --- a/ext/gl/gstglbasemixer.c +++ b/ext/gl/gstglbasemixer.c @@ -489,11 +489,31 @@ gst_gl_base_mixer_decide_allocation (GstGLBaseMixer * mix, GstQuery * query) GST_OBJECT_UNLOCK (mix->display); } + { + GstGLAPI current_gl_api = gst_gl_context_get_gl_api (mix->context); + if ((current_gl_api & mix_class->supported_gl_api) == 0) + goto unsupported_gl_api; + } + if (mix_class->decide_allocation) ret = mix_class->decide_allocation (mix, query); return ret; +unsupported_gl_api: + { + GstGLAPI gl_api = gst_gl_context_get_gl_api (mix->context); + gchar *gl_api_str = gst_gl_api_to_string (gl_api); + gchar *supported_gl_api_str = + gst_gl_api_to_string (mix_class->supported_gl_api); + GST_ELEMENT_ERROR (mix, RESOURCE, BUSY, + ("GL API's not compatible context: %s supported: %s", gl_api_str, + supported_gl_api_str), (NULL)); + + g_free (supported_gl_api_str); + g_free (gl_api_str); + return FALSE; + } context_error: { GST_ELEMENT_ERROR (mix, RESOURCE, NOT_FOUND, ("%s", error->message), From a8862ac44053ca0e0750d8140cf4a1e4300264bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wang=20Xin-yu=20=28=E7=8E=8B=E6=98=95=E5=AE=87=29?= Date: Wed, 24 Feb 2016 10:45:17 +0800 Subject: [PATCH 239/381] glmixer: iterator didn't advance in continue statement Leading to a deadlock. https://bugzilla.gnome.org/show_bug.cgi?id=760873 --- ext/gl/gstglmosaic.c | 1 + ext/gl/gstglstereomix.c | 1 + ext/gl/gstglvideomixer.c | 2 ++ 3 files changed, 4 insertions(+) diff --git a/ext/gl/gstglmosaic.c b/ext/gl/gstglmosaic.c index 83ac79810e..377f71869f 100644 --- a/ext/gl/gstglmosaic.c +++ b/ext/gl/gstglmosaic.c @@ -299,6 +299,7 @@ gst_gl_mosaic_callback (gpointer stuff) GST_DEBUG ("skipping texture:%u pad:%p width:%u height %u", in_tex, pad, width, height); count++; + walk = g_list_next (walk); continue; } diff --git a/ext/gl/gstglstereomix.c b/ext/gl/gstglstereomix.c index fee1080c26..ed604b2c81 100644 --- a/ext/gl/gstglstereomix.c +++ b/ext/gl/gstglstereomix.c @@ -505,6 +505,7 @@ gst_gl_stereo_mix_process_frames (GstGLStereoMix * mixer) if (!pad || !pad->current_buffer) { GST_DEBUG ("skipping texture, null frame"); + walk = g_list_next (walk); continue; } diff --git a/ext/gl/gstglvideomixer.c b/ext/gl/gstglvideomixer.c index e48151bd04..72ad8c232d 100644 --- a/ext/gl/gstglvideomixer.c +++ b/ext/gl/gstglvideomixer.c @@ -1437,11 +1437,13 @@ gst_gl_video_mixer_callback (gpointer stuff) || pad->alpha == 0.0f) { GST_DEBUG ("skipping texture:%u pad:%p width:%u height:%u alpha:%f", mix_pad->current_texture, pad, in_width, in_height, pad->alpha); + walk = g_list_next (walk); continue; } if (!_set_blend_state (video_mixer, pad)) { GST_FIXME_OBJECT (pad, "skipping due to incorrect blend parameters"); + walk = g_list_next (walk); continue; } From 86d21e94553ad72f8fb3e0281555663539bea921 Mon Sep 17 00:00:00 2001 From: Thiago Santos Date: Thu, 18 Feb 2016 10:57:51 -0300 Subject: [PATCH 240/381] videoaggregator: fix caps queries to allow proper renegotiation When caps are already negotiated it should be possible to select formats other than the one that was negotiated. If downstream allows alpha video caps and it has already negotiated to a non-alpha format, caps queries should still return the alpha caps as a possible format as caps renegotiation can happen. Includes tests (for compositor) to check that caps queries done after a caps has been negotiated returns complete results https://bugzilla.gnome.org/show_bug.cgi?id=757610 --- gst-libs/gst/video/gstvideoaggregator.c | 10 +- tests/check/elements/compositor.c | 211 ++++++++++++++++++++++++ 2 files changed, 212 insertions(+), 9 deletions(-) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index 2b0c87630e..1bf6ea0b20 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -892,15 +892,7 @@ gst_videoaggregator_pad_sink_getcaps (GstPad * pad, GstVideoAggregator * vagg, GST_DEBUG_OBJECT (pad, "Get caps with filter: %" GST_PTR_FORMAT, filter); - srccaps = gst_pad_get_current_caps (srcpad); - if (srccaps == NULL) { - srccaps = gst_pad_peer_query_caps (srcpad, template_caps); - GST_DEBUG_OBJECT (pad, "No output caps, using possible formats: %" - GST_PTR_FORMAT, srccaps); - } else { - GST_DEBUG_OBJECT (pad, "Using output caps: %" GST_PTR_FORMAT, srccaps); - } - + srccaps = gst_pad_peer_query_caps (srcpad, template_caps); srccaps = gst_caps_make_writable (srccaps); has_alpha = gst_videoaggregator_caps_has_alpha (srccaps); diff --git a/tests/check/elements/compositor.c b/tests/check/elements/compositor.c index 77c65cfed1..cf50e15321 100644 --- a/tests/check/elements/compositor.c +++ b/tests/check/elements/compositor.c @@ -245,6 +245,216 @@ GST_START_TEST (test_event) GST_END_TEST; +static GstBuffer * +create_video_buffer (GstCaps * caps, gint ts_in_seconds) +{ + GstVideoInfo info; + guint size; + GstBuffer *buf; + GstMapInfo mapinfo; + + fail_unless (gst_video_info_from_caps (&info, caps)); + + size = GST_VIDEO_INFO_WIDTH (&info) * GST_VIDEO_INFO_HEIGHT (&info); + + switch (GST_VIDEO_INFO_FORMAT (&info)) { + case GST_VIDEO_FORMAT_RGB: + size *= 3; + break; + case GST_VIDEO_FORMAT_RGBA: + case GST_VIDEO_FORMAT_ARGB: + size *= 4; + break; + case GST_VIDEO_FORMAT_I420: + size *= 2; + break; + default: + fail ("Unsupported test format"); + } + + buf = gst_buffer_new_and_alloc (size); + /* write something to avoid uninitialized error issues (valgrind) */ + gst_buffer_map (buf, &mapinfo, GST_MAP_WRITE); + memset (mapinfo.data, 0, mapinfo.size); + gst_buffer_unmap (buf, &mapinfo); + + GST_BUFFER_PTS (buf) = ts_in_seconds * GST_SECOND; + GST_BUFFER_DURATION (buf) = GST_SECOND; + return buf; +} + +#define MODE_ALL 1 +#define MODE_NON_ALPHA 2 + +/* mostly copied from videoaggregator */ +static inline GstCaps * +_get_non_alpha_caps_from_caps (GstCaps * caps) +{ + GstCaps *result; + guint i, size; + + size = gst_caps_get_size (caps); + result = gst_caps_new_empty (); + for (i = 0; i < size; i++) { + GstStructure *s = gst_caps_get_structure (caps, i); + const GValue *formats = gst_structure_get_value (s, "format"); + GValue new_formats = { 0, }; + gboolean has_format = FALSE; + + /* FIXME what to do if formats are missing? */ + if (formats) { + const GstVideoFormatInfo *info; + + if (GST_VALUE_HOLDS_LIST (formats)) { + guint list_size = gst_value_list_get_size (formats); + guint index; + + g_value_init (&new_formats, GST_TYPE_LIST); + + for (index = 0; index < list_size; index++) { + const GValue *list_item = gst_value_list_get_value (formats, index); + + info = + gst_video_format_get_info (gst_video_format_from_string + (g_value_get_string (list_item))); + if (!GST_VIDEO_FORMAT_INFO_HAS_ALPHA (info)) { + has_format = TRUE; + gst_value_list_append_value (&new_formats, list_item); + } + } + + } else if (G_VALUE_HOLDS_STRING (formats)) { + info = + gst_video_format_get_info (gst_video_format_from_string + (g_value_get_string (formats))); + if (!GST_VIDEO_FORMAT_INFO_HAS_ALPHA (info)) { + has_format = TRUE; + gst_value_init_and_copy (&new_formats, formats); + } + + } else { + g_assert_not_reached (); + GST_WARNING ("Unexpected type for video 'format' field: %s", + G_VALUE_TYPE_NAME (formats)); + } + + if (has_format) { + s = gst_structure_copy (s); + gst_structure_take_value (s, "format", &new_formats); + gst_caps_append_structure (result, s); + } + + } + } + + return result; +} + +static void +run_late_caps_query_test (GstCaps * input_caps, GstCaps * output_allowed_caps, + gint expected_caps_mode) +{ + GstElement *compositor, *capsfilter, *sink; + GstElement *pipeline; + gboolean res; + GstStateChangeReturn state_res; + GstPad *srcpad1, *srcpad2; + GstPad *sinkpad1, *sinkpad2; + GstSegment segment; + GstCaps *caps, *all_caps, *non_alpha_caps; + + all_caps = + gst_caps_from_string (GST_VIDEO_CAPS_MAKE + (" { AYUV, BGRA, ARGB, RGBA, ABGR, Y444, Y42B, YUY2, UYVY, " + " YVYU, I420, YV12, NV12, NV21, Y41B, RGB, BGR, xRGB, xBGR, " + " RGBx, BGRx } ")); + non_alpha_caps = + gst_caps_from_string (GST_VIDEO_CAPS_MAKE + (" { Y444, Y42B, YUY2, UYVY, " + " YVYU, I420, YV12, NV12, NV21, Y41B, RGB, BGR, xRGB, xBGR, " + " RGBx, BGRx } ")); + + compositor = gst_element_factory_make ("compositor", "compositor"); + capsfilter = gst_element_factory_make ("capsfilter", "out-cf"); + sink = gst_element_factory_make ("fakesink", "sink"); + pipeline = gst_pipeline_new ("test-pipeline"); + + gst_bin_add_many (GST_BIN (pipeline), compositor, capsfilter, sink, NULL); + res = gst_element_link (compositor, capsfilter); + fail_unless (res == TRUE, NULL); + res = gst_element_link (capsfilter, sink); + fail_unless (res == TRUE, NULL); + + sinkpad1 = gst_element_get_request_pad (compositor, "sink_%u"); + srcpad1 = gst_pad_new ("src1", GST_PAD_SRC); + fail_unless (gst_pad_link (srcpad1, sinkpad1) == GST_PAD_LINK_OK); + gst_pad_set_active (srcpad1, TRUE); + + state_res = gst_element_set_state (pipeline, GST_STATE_PLAYING); + fail_if (state_res == GST_STATE_CHANGE_FAILURE); + + if (output_allowed_caps) + g_object_set (capsfilter, "caps", output_allowed_caps, NULL); + + gst_segment_init (&segment, GST_FORMAT_TIME); + fail_unless (gst_pad_push_event (srcpad1, + gst_event_new_stream_start ("test-1"))); + fail_unless (gst_pad_push_event (srcpad1, gst_event_new_caps (input_caps))); + fail_unless (gst_pad_push_event (srcpad1, gst_event_new_segment (&segment))); + fail_unless (gst_pad_push (srcpad1, + create_video_buffer (input_caps, 0)) == GST_FLOW_OK); + fail_unless (gst_pad_push (srcpad1, + create_video_buffer (input_caps, 1)) == GST_FLOW_OK); + + /* now comes the second pad */ + sinkpad2 = gst_element_get_request_pad (compositor, "sink_%u"); + srcpad2 = gst_pad_new ("src2", GST_PAD_SRC); + fail_unless (gst_pad_link (srcpad2, sinkpad2) == GST_PAD_LINK_OK); + gst_pad_set_active (srcpad2, TRUE); + fail_unless (gst_pad_push_event (srcpad2, + gst_event_new_stream_start ("test-2"))); + + caps = gst_pad_peer_query_caps (srcpad2, NULL); + fail_unless (gst_caps_is_equal (caps, + expected_caps_mode == MODE_ALL ? all_caps : non_alpha_caps)); + gst_caps_unref (caps); + + gst_pad_set_active (srcpad1, FALSE); + gst_pad_set_active (srcpad2, FALSE); + gst_element_set_state (pipeline, GST_STATE_NULL); + gst_element_release_request_pad (compositor, sinkpad1); + gst_element_release_request_pad (compositor, sinkpad2); + gst_object_unref (sinkpad1); + gst_object_unref (sinkpad2); + gst_object_unref (pipeline); + gst_object_unref (srcpad1); + gst_object_unref (srcpad2); + gst_caps_unref (all_caps); + gst_caps_unref (non_alpha_caps); +} + +GST_START_TEST (test_late_caps_query) +{ + GstCaps *rgb_caps; + GstCaps *non_alpha_caps; + + rgb_caps = gst_caps_from_string ("video/x-raw, format=(string)RGB, " + "width=(int)100, height=(int)100, framerate=(fraction)1/1"); + + non_alpha_caps = gst_caps_from_string ("video/x-raw, format=(string)RGB"); + + /* check that a 2nd pad that is added late to compositor will be able to + * negotiate to formats that depend only on downstream caps and not on what + * the other pads have already negotiated */ + run_late_caps_query_test (rgb_caps, NULL, MODE_ALL); + run_late_caps_query_test (rgb_caps, non_alpha_caps, MODE_NON_ALPHA); + + gst_caps_unref (non_alpha_caps); + gst_caps_unref (rgb_caps); +} + +GST_END_TEST; + static guint play_count = 0; static GstEvent *play_seek_event = NULL; @@ -1660,6 +1870,7 @@ compositor_suite (void) suite_add_tcase (s, tc_chain); tcase_add_test (tc_chain, test_caps); tcase_add_test (tc_chain, test_event); + tcase_add_test (tc_chain, test_late_caps_query); tcase_add_test (tc_chain, test_play_twice); tcase_add_test (tc_chain, test_play_twice_then_add_and_play_again); tcase_add_test (tc_chain, test_add_pad); From ea7e09930770a296a43ef755682205f9756018f6 Mon Sep 17 00:00:00 2001 From: Thiago Santos Date: Tue, 23 Feb 2016 12:17:59 -0300 Subject: [PATCH 241/381] tests: compositor: add tests for caps queries Verifies that proper caps are returned based on what downstream restricts. --- tests/check/elements/compositor.c | 159 ++++++++++++++++-------------- 1 file changed, 85 insertions(+), 74 deletions(-) diff --git a/tests/check/elements/compositor.c b/tests/check/elements/compositor.c index cf50e15321..0f5aad4a51 100644 --- a/tests/check/elements/compositor.c +++ b/tests/check/elements/compositor.c @@ -45,6 +45,24 @@ static GMainLoop *main_loop; +static GstCaps * +_compositor_get_all_supported_caps (void) +{ + return gst_caps_from_string (GST_VIDEO_CAPS_MAKE + (" { AYUV, BGRA, ARGB, RGBA, ABGR, Y444, Y42B, YUY2, UYVY, " + " YVYU, I420, YV12, NV12, NV21, Y41B, RGB, BGR, xRGB, xBGR, " + " RGBx, BGRx } ")); +} + +static GstCaps * +_compositor_get_non_alpha_supported_caps (void) +{ + return gst_caps_from_string (GST_VIDEO_CAPS_MAKE + (" { Y444, Y42B, YUY2, UYVY, " + " YVYU, I420, YV12, NV12, NV21, Y41B, RGB, BGR, xRGB, xBGR, " + " RGBx, BGRx } ")); +} + /* make sure downstream gets a CAPS event before buffers are sent */ GST_START_TEST (test_caps) { @@ -283,73 +301,73 @@ create_video_buffer (GstCaps * caps, gint ts_in_seconds) return buf; } + +GST_START_TEST (test_caps_query) +{ + GstElement *compositor, *capsfilter, *sink; + GstElement *pipeline; + gboolean res; + GstStateChangeReturn state_res; + GstPad *sinkpad; + GstCaps *caps, *restriction_caps; + GstCaps *all_caps, *non_alpha_caps; + + /* initial setup */ + all_caps = _compositor_get_all_supported_caps (); + non_alpha_caps = _compositor_get_non_alpha_supported_caps (); + + compositor = gst_element_factory_make ("compositor", "compositor"); + capsfilter = gst_element_factory_make ("capsfilter", "out-cf"); + sink = gst_element_factory_make ("fakesink", "sink"); + pipeline = gst_pipeline_new ("test-pipeline"); + + gst_bin_add_many (GST_BIN (pipeline), compositor, capsfilter, sink, NULL); + res = gst_element_link (compositor, capsfilter); + fail_unless (res == TRUE, NULL); + res = gst_element_link (capsfilter, sink); + fail_unless (res == TRUE, NULL); + + sinkpad = gst_element_get_request_pad (compositor, "sink_%u"); + + state_res = gst_element_set_state (pipeline, GST_STATE_PLAYING); + fail_if (state_res == GST_STATE_CHANGE_FAILURE); + + /* try an unrestricted caps query, should return all formats */ + caps = gst_pad_query_caps (sinkpad, NULL); + fail_unless (gst_caps_is_equal (caps, all_caps)); + gst_caps_unref (caps); + + /* now restrict downstream to a single alpha format, it should still + * be able to convert anything else to it */ + restriction_caps = gst_caps_from_string ("video/x-raw, format=(string)AYUV"); + g_object_set (capsfilter, "caps", restriction_caps, NULL); + caps = gst_pad_query_caps (sinkpad, NULL); + fail_unless (gst_caps_is_equal (caps, all_caps)); + gst_caps_unref (caps); + gst_caps_unref (restriction_caps); + + /* now restrict downstream to a non-alpha format, it should + * be able to accept non-alpha formats */ + restriction_caps = gst_caps_from_string ("video/x-raw, format=(string)I420"); + g_object_set (capsfilter, "caps", restriction_caps, NULL); + caps = gst_pad_query_caps (sinkpad, NULL); + fail_unless (gst_caps_is_equal (caps, non_alpha_caps)); + gst_caps_unref (caps); + gst_caps_unref (restriction_caps); + + gst_element_set_state (pipeline, GST_STATE_NULL); + gst_element_release_request_pad (compositor, sinkpad); + gst_object_unref (sinkpad); + gst_object_unref (pipeline); + gst_caps_unref (all_caps); + gst_caps_unref (non_alpha_caps); +} + +GST_END_TEST; + #define MODE_ALL 1 #define MODE_NON_ALPHA 2 -/* mostly copied from videoaggregator */ -static inline GstCaps * -_get_non_alpha_caps_from_caps (GstCaps * caps) -{ - GstCaps *result; - guint i, size; - - size = gst_caps_get_size (caps); - result = gst_caps_new_empty (); - for (i = 0; i < size; i++) { - GstStructure *s = gst_caps_get_structure (caps, i); - const GValue *formats = gst_structure_get_value (s, "format"); - GValue new_formats = { 0, }; - gboolean has_format = FALSE; - - /* FIXME what to do if formats are missing? */ - if (formats) { - const GstVideoFormatInfo *info; - - if (GST_VALUE_HOLDS_LIST (formats)) { - guint list_size = gst_value_list_get_size (formats); - guint index; - - g_value_init (&new_formats, GST_TYPE_LIST); - - for (index = 0; index < list_size; index++) { - const GValue *list_item = gst_value_list_get_value (formats, index); - - info = - gst_video_format_get_info (gst_video_format_from_string - (g_value_get_string (list_item))); - if (!GST_VIDEO_FORMAT_INFO_HAS_ALPHA (info)) { - has_format = TRUE; - gst_value_list_append_value (&new_formats, list_item); - } - } - - } else if (G_VALUE_HOLDS_STRING (formats)) { - info = - gst_video_format_get_info (gst_video_format_from_string - (g_value_get_string (formats))); - if (!GST_VIDEO_FORMAT_INFO_HAS_ALPHA (info)) { - has_format = TRUE; - gst_value_init_and_copy (&new_formats, formats); - } - - } else { - g_assert_not_reached (); - GST_WARNING ("Unexpected type for video 'format' field: %s", - G_VALUE_TYPE_NAME (formats)); - } - - if (has_format) { - s = gst_structure_copy (s); - gst_structure_take_value (s, "format", &new_formats); - gst_caps_append_structure (result, s); - } - - } - } - - return result; -} - static void run_late_caps_query_test (GstCaps * input_caps, GstCaps * output_allowed_caps, gint expected_caps_mode) @@ -363,16 +381,8 @@ run_late_caps_query_test (GstCaps * input_caps, GstCaps * output_allowed_caps, GstSegment segment; GstCaps *caps, *all_caps, *non_alpha_caps; - all_caps = - gst_caps_from_string (GST_VIDEO_CAPS_MAKE - (" { AYUV, BGRA, ARGB, RGBA, ABGR, Y444, Y42B, YUY2, UYVY, " - " YVYU, I420, YV12, NV12, NV21, Y41B, RGB, BGR, xRGB, xBGR, " - " RGBx, BGRx } ")); - non_alpha_caps = - gst_caps_from_string (GST_VIDEO_CAPS_MAKE - (" { Y444, Y42B, YUY2, UYVY, " - " YVYU, I420, YV12, NV12, NV21, Y41B, RGB, BGR, xRGB, xBGR, " - " RGBx, BGRx } ")); + all_caps = _compositor_get_all_supported_caps (); + non_alpha_caps = _compositor_get_non_alpha_supported_caps (); compositor = gst_element_factory_make ("compositor", "compositor"); capsfilter = gst_element_factory_make ("capsfilter", "out-cf"); @@ -1870,6 +1880,7 @@ compositor_suite (void) suite_add_tcase (s, tc_chain); tcase_add_test (tc_chain, test_caps); tcase_add_test (tc_chain, test_event); + tcase_add_test (tc_chain, test_caps_query); tcase_add_test (tc_chain, test_late_caps_query); tcase_add_test (tc_chain, test_play_twice); tcase_add_test (tc_chain, test_play_twice_then_add_and_play_again); From 3d0f8cfc21d4bccdde5de84f6c68f4d6468b174c Mon Sep 17 00:00:00 2001 From: Thiago Santos Date: Tue, 23 Feb 2016 12:42:19 -0300 Subject: [PATCH 242/381] tests: compositor: drop special case for valgrind timeout The default one is 6 minutes, the test was using 5 minutes so just resort to using the default. For the non-valgrind test also use the default 20 secs instead of reducing it to 6s. No real reason to set a custom value here. --- tests/check/elements/compositor.c | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/tests/check/elements/compositor.c b/tests/check/elements/compositor.c index 0f5aad4a51..8b5fd5b26a 100644 --- a/tests/check/elements/compositor.c +++ b/tests/check/elements/compositor.c @@ -1902,17 +1902,6 @@ compositor_suite (void) tcase_add_test (tc_chain, test_start_time_first_live_drop_3); tcase_add_test (tc_chain, test_start_time_first_live_drop_3_unlinked_1); - /* Use a longer timeout */ -#ifdef HAVE_VALGRIND - if (RUNNING_ON_VALGRIND) { - tcase_set_timeout (tc_chain, 5 * 60); - } else -#endif - { - /* this is shorter than the default 60 seconds?! (tpm) */ - /* tcase_set_timeout (tc_chain, 6); */ - } - return s; } From ca15a26c11cf9a39d353d693077f12ebd3142d56 Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Tue, 8 Mar 2016 02:06:46 +1100 Subject: [PATCH 243/381] glvideomixer: signal continuation in reset We want to iterate over all the pads, not just the first one. Fix by returning TRUE in the GstAggregatorPadForeachFunc. Removes a GST_IS_GL_CONTEXT() assertion on shutdown with >2 inputs using gst-launch. --- ext/gl/gstglvideomixer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/gl/gstglvideomixer.c b/ext/gl/gstglvideomixer.c index 72ad8c232d..208a6df83d 100644 --- a/ext/gl/gstglvideomixer.c +++ b/ext/gl/gstglvideomixer.c @@ -1083,7 +1083,7 @@ _reset_pad_gl (GstAggregator * agg, GstAggregatorPad * aggpad, gpointer udata) pad->vertex_buffer = 0; } - return FALSE; + return TRUE; } static void From 81d724a32534d859268c1a66a29d2078d7c27159 Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Wed, 16 Mar 2016 22:16:34 +1100 Subject: [PATCH 244/381] glstereo{mix,split}: allow running on GLES 2/3 It's mostly supported for GLES 2.x, fully supported on GLES 3.x --- ext/gl/gstglstereomix.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ext/gl/gstglstereomix.c b/ext/gl/gstglstereomix.c index ed604b2c81..989d7f61ee 100644 --- a/ext/gl/gstglstereomix.c +++ b/ext/gl/gstglstereomix.c @@ -167,7 +167,8 @@ gst_gl_stereo_mix_class_init (GstGLStereoMixClass * klass) gst_gl_stereo_mix_get_output_buffer; videoaggregator_class->find_best_format = gst_gl_stereo_mix_find_best_format; - base_mix_class->supported_gl_api = GST_GL_API_OPENGL | GST_GL_API_OPENGL3; + base_mix_class->supported_gl_api = + GST_GL_API_GLES2 | GST_GL_API_OPENGL | GST_GL_API_OPENGL3; } static void From 87418066718860665daaa2487ba42038416ae57f Mon Sep 17 00:00:00 2001 From: Vineeth TM Date: Fri, 4 Mar 2016 15:50:26 +0900 Subject: [PATCH 245/381] bad: use new gst_element_class_add_static_pad_template() https://bugzilla.gnome.org/show_bug.cgi?id=763081 --- ext/gl/gstglmixer.c | 6 ++---- ext/gl/gstglmixerbin.c | 3 +-- ext/gl/gstglstereomix.c | 6 ++---- gst/compositor/compositor.c | 6 ++---- 4 files changed, 7 insertions(+), 14 deletions(-) diff --git a/ext/gl/gstglmixer.c b/ext/gl/gstglmixer.c index 5ca1316ca0..7059b4837b 100644 --- a/ext/gl/gstglmixer.c +++ b/ext/gl/gstglmixer.c @@ -380,10 +380,8 @@ gst_gl_mixer_class_init (GstGLMixerClass * klass) gobject_class->get_property = gst_gl_mixer_get_property; gobject_class->set_property = gst_gl_mixer_set_property; - gst_element_class_add_pad_template (element_class, - gst_static_pad_template_get (&src_factory)); - gst_element_class_add_pad_template (element_class, - gst_static_pad_template_get (&sink_factory)); + gst_element_class_add_static_pad_template (element_class, &src_factory); + gst_element_class_add_static_pad_template (element_class, &sink_factory); agg_class->sinkpads_type = GST_TYPE_GL_MIXER_PAD; agg_class->sink_query = gst_gl_mixer_sink_query; diff --git a/ext/gl/gstglmixerbin.c b/ext/gl/gstglmixerbin.c index 6f83f82a29..bb74ef9139 100644 --- a/ext/gl/gstglmixerbin.c +++ b/ext/gl/gstglmixerbin.c @@ -169,8 +169,7 @@ gst_gl_mixer_bin_class_init (GstGLMixerBinClass * klass) G_SIGNAL_RUN_LAST, 0, NULL, NULL, g_cclosure_marshal_generic, GST_TYPE_ELEMENT, 0); - gst_element_class_add_pad_template (element_class, - gst_static_pad_template_get (&src_factory)); + gst_element_class_add_static_pad_template (element_class, &src_factory); upload_caps = gst_gl_upload_get_input_template_caps (); gst_element_class_add_pad_template (element_class, diff --git a/ext/gl/gstglstereomix.c b/ext/gl/gstglstereomix.c index 989d7f61ee..663c1b744a 100644 --- a/ext/gl/gstglstereomix.c +++ b/ext/gl/gstglstereomix.c @@ -150,10 +150,8 @@ gst_gl_stereo_mix_class_init (GstGLStereoMixClass * klass) GST_TYPE_GL_STEREO_DOWNMIX_MODE_TYPE, DEFAULT_DOWNMIX, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); - gst_element_class_add_pad_template (element_class, - gst_static_pad_template_get (&src_factory)); - gst_element_class_add_pad_template (element_class, - gst_static_pad_template_get (&sink_factory)); + gst_element_class_add_static_pad_template (element_class, &src_factory); + gst_element_class_add_static_pad_template (element_class, &sink_factory); agg_class->sinkpads_type = GST_TYPE_GL_STEREO_MIX_PAD; agg_class->stop = gst_gl_stereo_mix_stop; diff --git a/gst/compositor/compositor.c b/gst/compositor/compositor.c index c5b1a9abf4..1c0b39bcfe 100644 --- a/gst/compositor/compositor.c +++ b/gst/compositor/compositor.c @@ -1103,10 +1103,8 @@ gst_compositor_class_init (GstCompositorClass * klass) GST_TYPE_COMPOSITOR_BACKGROUND, DEFAULT_BACKGROUND, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); - gst_element_class_add_pad_template (gstelement_class, - gst_static_pad_template_get (&src_factory)); - gst_element_class_add_pad_template (gstelement_class, - gst_static_pad_template_get (&sink_factory)); + gst_element_class_add_static_pad_template (gstelement_class, &src_factory); + gst_element_class_add_static_pad_template (gstelement_class, &sink_factory); gst_element_class_set_static_metadata (gstelement_class, "Compositor", "Filter/Editor/Video/Compositor", From 3c7906c13b1e537dcc346422be011e9f0f5584b7 Mon Sep 17 00:00:00 2001 From: Nirbheek Chauhan Date: Mon, 28 Mar 2016 08:45:45 +0530 Subject: [PATCH 246/381] tests/compositor: Add test for aggregator pad numbering Tests that the behaviour in 7a5cb5a473 is being conformed to. --- tests/check/elements/compositor.c | 35 +++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/tests/check/elements/compositor.c b/tests/check/elements/compositor.c index 8b5fd5b26a..6e7fe22d73 100644 --- a/tests/check/elements/compositor.c +++ b/tests/check/elements/compositor.c @@ -1708,6 +1708,40 @@ GST_START_TEST (test_pad_z_order) GST_END_TEST; +/* + * Test that the pad numbering assigned by aggregator behaves as follows: + * 1. If a pad number is requested, it must be assigned if it is available + * 2. When numbering automatically, the largest available pad number is used + * 3. Pad names must be unique + */ +GST_START_TEST (test_pad_numbering) +{ + GstElement *mixer; + GstPad *sinkpad1, *sinkpad2, *sinkpad3, *sinkpad4; + + GST_INFO ("preparing test"); + + mixer = gst_element_factory_make ("compositor", NULL); + sinkpad1 = gst_element_get_request_pad (mixer, "sink_%u"); + sinkpad2 = gst_element_get_request_pad (mixer, "sink_7"); + sinkpad3 = gst_element_get_request_pad (mixer, "sink_1"); + sinkpad4 = gst_element_get_request_pad (mixer, "sink_%u"); + + ck_assert_str_eq (GST_PAD_NAME (sinkpad1), "sink_0"); + ck_assert_str_eq (GST_PAD_NAME (sinkpad2), "sink_7"); + ck_assert_str_eq (GST_PAD_NAME (sinkpad3), "sink_1"); + ck_assert_str_eq (GST_PAD_NAME (sinkpad4), "sink_8"); + + /* cleanup */ + gst_object_unref (mixer); + gst_object_unref (sinkpad1); + gst_object_unref (sinkpad2); + gst_object_unref (sinkpad3); + gst_object_unref (sinkpad4); +} + +GST_END_TEST; + typedef struct { gint buffers_sent; @@ -1895,6 +1929,7 @@ compositor_suite (void) tcase_add_test (tc_chain, test_obscured_skipped); tcase_add_test (tc_chain, test_ignore_eos); tcase_add_test (tc_chain, test_pad_z_order); + tcase_add_test (tc_chain, test_pad_numbering); tcase_add_test (tc_chain, test_start_time_zero_live_drop_0); tcase_add_test (tc_chain, test_start_time_zero_live_drop_3); tcase_add_test (tc_chain, test_start_time_zero_live_drop_3_unlinked_1); From efc12994262ab3e9c54e0fa2ce46f3ea8226c4c8 Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Mon, 4 Apr 2016 13:43:30 +1000 Subject: [PATCH 247/381] glmixerbin: proxy the start-time-* properties from aggregator --- ext/gl/gstglmixerbin.c | 45 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/ext/gl/gstglmixerbin.c b/ext/gl/gstglmixerbin.c index bb74ef9139..1e6ebeb14e 100644 --- a/ext/gl/gstglmixerbin.c +++ b/ext/gl/gstglmixerbin.c @@ -31,6 +31,36 @@ GST_DEBUG_CATEGORY (gst_gl_mixer_bin_debug); #define DEFAULT_LATENCY 0 +#define DEFAULT_START_TIME_SELECTION 0 +#define DEFAULT_START_TIME (-1) + +typedef enum +{ + GST_GL_MIXER_BIN_START_TIME_SELECTION_ZERO, + GST_GL_MIXER_BIN_START_TIME_SELECTION_FIRST, + GST_GL_MIXER_BIN_START_TIME_SELECTION_SET +} GstGLMixerBinStartTimeSelection; + +static GType +gst_gl_mixer_bin_start_time_selection_get_type (void) +{ + static GType gtype = 0; + + if (gtype == 0) { + static const GEnumValue values[] = { + {GST_GL_MIXER_BIN_START_TIME_SELECTION_ZERO, + "Start at 0 running time (default)", "zero"}, + {GST_GL_MIXER_BIN_START_TIME_SELECTION_FIRST, + "Start at first observed input running time", "first"}, + {GST_GL_MIXER_BIN_START_TIME_SELECTION_SET, + "Set start time with start-time property", "set"}, + {0, NULL, NULL} + }; + + gtype = g_enum_register_static ("GstGLMixerBinStartTimeSelection", values); + } + return gtype; +} struct input_chain { @@ -82,6 +112,8 @@ enum PROP_0, PROP_MIXER, PROP_LATENCY, + PROP_START_TIME_SELECTION, + PROP_START_TIME, }; enum @@ -156,6 +188,19 @@ gst_gl_mixer_bin_class_init (GstGLMixerBinClass * klass) (G_MAXLONG == G_MAXINT64) ? G_MAXINT64 : (G_MAXLONG * GST_SECOND - 1), DEFAULT_LATENCY, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + g_object_class_install_property (gobject_class, PROP_START_TIME_SELECTION, + g_param_spec_enum ("start-time-selection", "Start Time Selection", + "Decides which start time is output", + gst_gl_mixer_bin_start_time_selection_get_type (), + DEFAULT_START_TIME_SELECTION, + G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + + g_object_class_install_property (gobject_class, PROP_START_TIME, + g_param_spec_uint64 ("start-time", "Start Time", + "Start time to use if start-time-selection=set", 0, + G_MAXUINT64, + DEFAULT_START_TIME, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + /** * GstMixerBin::create-element: * @object: the #GstGLMixerBin From 71877db25ffdc29fd3874e9efc455a7dd6999fe4 Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Tue, 5 Apr 2016 16:22:49 +1000 Subject: [PATCH 248/381] glmixer: set the current texture to 0 before mapping If we fail mapping, we don't want to use undefined video data in the subclass. --- ext/gl/gstglmixer.c | 1 + 1 file changed, 1 insertion(+) diff --git a/ext/gl/gstglmixer.c b/ext/gl/gstglmixer.c index 7059b4837b..3fc7fce2c8 100644 --- a/ext/gl/gstglmixer.c +++ b/ext/gl/gstglmixer.c @@ -604,6 +604,7 @@ _upload_frames (GstAggregator * agg, GstAggregatorPad * agg_pad, GstGLMixerPad *pad = GST_GL_MIXER_PAD (agg_pad); GstGLMixer *mix = GST_GL_MIXER (agg); + pad->current_texture = 0; if (vaggpad->buffer != NULL) { GstVideoInfo gl_info; GstVideoFrame gl_frame; From f2c2717ac7c217b2448303a7d8ac2d855f88236e Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Wed, 23 Mar 2016 03:16:11 +0000 Subject: [PATCH 249/381] glvideomixer: add support for the affine transformation meta --- ext/gl/gstglvideomixer.c | 41 +++++++++++++++++++++++++++------------- 1 file changed, 28 insertions(+), 13 deletions(-) diff --git a/ext/gl/gstglvideomixer.c b/ext/gl/gstglvideomixer.c index 208a6df83d..fddd4cbbeb 100644 --- a/ext/gl/gstglvideomixer.c +++ b/ext/gl/gstglvideomixer.c @@ -43,6 +43,8 @@ #include "config.h" #endif +#include + #include "gstglvideomixer.h" #include "gstglmixerbin.h" @@ -472,15 +474,6 @@ static gboolean gst_gl_video_mixer_process_textures (GstGLMixer * mixer, static void gst_gl_video_mixer_callback (gpointer stuff); /* *INDENT-OFF* */ -/* vertex source */ -static const gchar *video_mixer_v_src = - "attribute vec4 a_position; \n" - "attribute vec2 a_texCoord; \n" - "varying vec2 v_texCoord; \n" - "void main() \n" - "{ \n" - " gl_Position = a_position; \n" - " v_texCoord = a_texCoord; \n" "}"; /* fragment source */ static const gchar *video_mixer_f_src = @@ -489,10 +482,10 @@ static const gchar *video_mixer_f_src = "#endif\n" "uniform sampler2D texture; \n" "uniform float alpha;\n" - "varying vec2 v_texCoord; \n" + "varying vec2 v_texcoord; \n" "void main() \n" "{ \n" - " vec4 rgba = texture2D( texture, v_texCoord );\n" + " vec4 rgba = texture2D(texture, v_texcoord);\n" " gl_FragColor = vec4(rgba.rgb, rgba.a * alpha);\n" "} \n"; @@ -1141,7 +1134,8 @@ gst_gl_video_mixer_init_shader (GstGLMixer * mixer, GstCaps * outcaps) video_mixer->shader); return gst_gl_context_gen_shader (GST_GL_BASE_MIXER (mixer)->context, - video_mixer_v_src, video_mixer_f_src, &video_mixer->shader); + gst_gl_shader_string_vertex_mat4_texture_transform, + video_mixer_f_src, &video_mixer->shader); } static gboolean @@ -1371,6 +1365,13 @@ _set_blend_state (GstGLVideoMixer * video_mixer, GstGLVideoMixerPad * mix_pad) return TRUE; } +static const gfloat identity_matrix[] = { + 1.0f, 0.0f, 0.0f, 0.0f, + 0.0f, 1.0f, 0.0f, 0.0f, + 0.0f, 0.0f, 1.0f, 0.0f, + 0.0f, 0.0f, 0.0f, 1.0f, +}; + /* opengl scene, params: input texture (not the output mixer->texture) */ static void gst_gl_video_mixer_callback (gpointer stuff) @@ -1407,7 +1408,7 @@ gst_gl_video_mixer_callback (gpointer stuff) attr_position_loc = gst_gl_shader_get_attribute_location (video_mixer->shader, "a_position"); attr_texture_loc = - gst_gl_shader_get_attribute_location (video_mixer->shader, "a_texCoord"); + gst_gl_shader_get_attribute_location (video_mixer->shader, "a_texcoord"); gl->Enable (GL_BLEND); @@ -1416,6 +1417,7 @@ gst_gl_video_mixer_callback (gpointer stuff) while (walk) { GstGLMixerPad *mix_pad = walk->data; GstGLVideoMixerPad *pad = walk->data; + GstVideoAggregatorPad *vagg_pad = walk->data; GstVideoInfo *v_info; guint in_tex; guint in_width, in_height; @@ -1495,6 +1497,19 @@ gst_gl_video_mixer_callback (gpointer stuff) gst_gl_shader_set_uniform_1i (video_mixer->shader, "texture", 0); gst_gl_shader_set_uniform_1f (video_mixer->shader, "alpha", pad->alpha); + { + GstVideoAffineTransformationMeta *af_meta; + + af_meta = + gst_buffer_get_video_affine_transformation_meta (vagg_pad->buffer); + if (af_meta) + gst_gl_shader_set_uniform_matrix_4fv (video_mixer->shader, + "u_transformation", 1, FALSE, af_meta->matrix); + else + gst_gl_shader_set_uniform_matrix_4fv (video_mixer->shader, + "u_transformation", 1, FALSE, identity_matrix); + } + gl->EnableVertexAttribArray (attr_position_loc); gl->EnableVertexAttribArray (attr_texture_loc); From 92245ec05f58129c0b2a8f439dbdbbe0ed75e9eb Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Mon, 4 Apr 2016 20:55:51 +1000 Subject: [PATCH 250/381] videoaggregator: repect the result of find_best_format in the default update_caps We weren't using the result of find_best_format at all. Also, move the find_best_format usage to the default update_caps() to make sure that it is also overridable. https://bugzilla.gnome.org/show_bug.cgi?id=764363 --- ext/gl/gstglvideomixer.c | 10 +-- gst-libs/gst/video/gstvideoaggregator.c | 101 ++++++++++++++++-------- 2 files changed, 69 insertions(+), 42 deletions(-) diff --git a/ext/gl/gstglvideomixer.c b/ext/gl/gstglvideomixer.c index fddd4cbbeb..ba3caa4fd1 100644 --- a/ext/gl/gstglvideomixer.c +++ b/ext/gl/gstglvideomixer.c @@ -977,14 +977,10 @@ _update_caps (GstVideoAggregator * vagg, GstCaps * caps, GstCaps * filter) { GstCaps *ret; - ret = - GST_VIDEO_AGGREGATOR_CLASS (gst_gl_video_mixer_parent_class)->update_caps - (vagg, caps, NULL); - if (filter) { - GstCaps *tmp = gst_caps_intersect (ret, filter); - gst_caps_unref (ret); - ret = tmp; + ret = gst_caps_intersect (caps, filter); + } else { + ret = gst_caps_ref (caps); } return ret; diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index 1bf6ea0b20..f9232e8dda 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -647,12 +647,42 @@ static GstCaps * gst_videoaggregator_default_update_caps (GstVideoAggregator * vagg, GstCaps * caps, GstCaps * filter) { - GstCaps *ret; + GstVideoAggregatorClass *vagg_klass = GST_VIDEO_AGGREGATOR_GET_CLASS (vagg); + GstCaps *ret, *best_format_caps; + gboolean at_least_one_alpha = FALSE; + GstVideoFormat best_format; + GstVideoInfo best_info; + + best_format = GST_VIDEO_FORMAT_UNKNOWN; + gst_video_info_init (&best_info); + + if (vagg_klass->find_best_format) { + vagg_klass->find_best_format (vagg, caps, &best_info, &at_least_one_alpha); + + best_format = GST_VIDEO_INFO_FORMAT (&best_info); + } + + if (best_format == GST_VIDEO_FORMAT_UNKNOWN) { + GstCaps *tmp = gst_caps_fixate (gst_caps_ref (caps)); + gst_video_info_from_caps (&best_info, tmp); + best_format = GST_VIDEO_INFO_FORMAT (&best_info); + gst_caps_unref (tmp); + } + + GST_DEBUG_OBJECT (vagg, + "The output format will now be : %d with chroma : %s", + best_format, gst_video_chroma_to_string (best_info.chroma_site)); + + best_format_caps = gst_caps_copy (caps); + gst_caps_set_simple (best_format_caps, "format", G_TYPE_STRING, + gst_video_format_to_string (best_format), "chroma-site", G_TYPE_STRING, + gst_video_chroma_to_string (best_info.chroma_site), NULL); + ret = gst_caps_merge (best_format_caps, gst_caps_ref (caps)); if (filter) { - ret = gst_caps_intersect (caps, filter); + ret = gst_caps_intersect (ret, filter); } else { - ret = gst_caps_ref (caps); + gst_caps_ref (ret); } return ret; @@ -667,15 +697,10 @@ gst_videoaggregator_update_src_caps (GstVideoAggregator * vagg) (GST_AGGREGATOR_GET_CLASS (vagg)->sinkpads_type); GstAggregator *agg = GST_AGGREGATOR (vagg); gboolean ret = TRUE, at_least_one_pad_configured = FALSE; - GstVideoFormat best_format; - GstVideoInfo best_info; gboolean at_least_one_alpha = FALSE; GstCaps *downstream_caps; GList *l; - best_format = GST_VIDEO_FORMAT_UNKNOWN; - gst_video_info_init (&best_info); - downstream_caps = gst_pad_get_allowed_caps (agg->srcpad); if (!downstream_caps || gst_caps_is_empty (downstream_caps)) { @@ -686,32 +711,6 @@ gst_videoaggregator_update_src_caps (GstVideoAggregator * vagg) return FALSE; } - if (vagg_klass->find_best_format) { - vagg_klass->find_best_format (vagg, downstream_caps, &best_info, - &at_least_one_alpha); - - best_format = GST_VIDEO_INFO_FORMAT (&best_info); - } - - if (best_format == GST_VIDEO_FORMAT_UNKNOWN) { - GstCaps *tmp = gst_caps_fixate (gst_caps_ref (downstream_caps)); - gst_video_info_from_caps (&best_info, tmp); - best_format = GST_VIDEO_INFO_FORMAT (&best_info); - gst_caps_unref (tmp); - } - - if (at_least_one_alpha - && !(best_info.finfo->flags & GST_VIDEO_FORMAT_FLAG_ALPHA)) { - GST_ELEMENT_ERROR (vagg, CORE, NEGOTIATION, - ("At least one of the input pads contains alpha, but downstream can't support alpha."), - ("Either convert your inputs to not contain alpha or add a videoconvert after the aggregator")); - return FALSE; - } - - GST_DEBUG_OBJECT (vagg, - "The output format will now be : %d with chroma : %s", - best_format, gst_video_chroma_to_string (best_info.chroma_site)); - GST_OBJECT_LOCK (vagg); for (l = GST_ELEMENT (vagg)->sinkpads; l; l = l->next) { GstVideoAggregatorPad *mpad = l->data; @@ -720,8 +719,10 @@ gst_videoaggregator_update_src_caps (GstVideoAggregator * vagg) || GST_VIDEO_INFO_HEIGHT (&mpad->info) == 0) continue; + if (mpad->info.finfo->flags & GST_VIDEO_FORMAT_FLAG_ALPHA) + at_least_one_alpha = TRUE; + at_least_one_pad_configured = TRUE; - break; } GST_OBJECT_UNLOCK (vagg); @@ -731,6 +732,9 @@ gst_videoaggregator_update_src_caps (GstVideoAggregator * vagg) peercaps = gst_pad_peer_query_caps (agg->srcpad, NULL); g_assert (vagg_klass->update_caps); + GST_DEBUG_OBJECT (vagg, "updating caps from %" GST_PTR_FORMAT, + downstream_caps); + GST_DEBUG_OBJECT (vagg, " with filter %" GST_PTR_FORMAT, peercaps); if (!(caps = vagg_klass->update_caps (vagg, downstream_caps, peercaps))) { GST_WARNING_OBJECT (vagg, "Subclass failed to update provided caps"); gst_caps_unref (downstream_caps); @@ -739,6 +743,8 @@ gst_videoaggregator_update_src_caps (GstVideoAggregator * vagg) ret = FALSE; goto done; } + GST_DEBUG_OBJECT (vagg, " to %" GST_PTR_FORMAT, caps); + gst_caps_unref (downstream_caps); if (peercaps) gst_caps_unref (peercaps); @@ -747,11 +753,36 @@ gst_videoaggregator_update_src_caps (GstVideoAggregator * vagg) g_assert (vagg_klass->fixate_caps); caps = gst_caps_make_writable (caps); + GST_DEBUG_OBJECT (vagg, "fixate caps from %" GST_PTR_FORMAT, caps); if (!(caps = vagg_klass->fixate_caps (vagg, caps))) { GST_WARNING_OBJECT (vagg, "Subclass failed to fixate provided caps"); ret = FALSE; goto done; } + GST_DEBUG_OBJECT (vagg, " to %" GST_PTR_FORMAT, caps); + } + + { + const GstVideoFormatInfo *finfo; + const gchar *v_format_str; + GstVideoFormat v_format; + GstStructure *s; + + s = gst_caps_get_structure (caps, 0); + v_format_str = gst_structure_get_string (s, "format"); + g_return_val_if_fail (v_format_str != NULL, FALSE); + v_format = gst_video_format_from_string (v_format_str); + g_return_val_if_fail (v_format != GST_VIDEO_FORMAT_UNKNOWN, FALSE); + finfo = gst_video_format_get_info (v_format); + g_return_val_if_fail (finfo != NULL, FALSE); + + if (at_least_one_alpha && !(finfo->flags & GST_VIDEO_FORMAT_FLAG_ALPHA)) { + GST_ELEMENT_ERROR (vagg, CORE, NEGOTIATION, + ("At least one of the input pads contains alpha, but configured caps don't support alpha."), + ("Either convert your inputs to not contain alpha or add a videoconvert after the aggregator")); + ret = FALSE; + goto done; + } } gst_video_info_from_caps (&vagg->info, caps); From 5e11ec7164276c738deac1fa3fe2c6bb3eaa0b55 Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Mon, 11 Apr 2016 16:43:45 +0000 Subject: [PATCH 251/381] glbasemixer: chain up to the parent implementation --- ext/gl/gstglbasemixer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/gl/gstglbasemixer.c b/ext/gl/gstglbasemixer.c index d207883776..c78ea8889a 100644 --- a/ext/gl/gstglbasemixer.c +++ b/ext/gl/gstglbasemixer.c @@ -665,7 +665,7 @@ gst_gl_base_mixer_set_property (GObject * object, static gboolean gst_gl_base_mixer_start (GstAggregator * agg) { - return TRUE; + return GST_AGGREGATOR_CLASS (parent_class)->start (agg);; } static gboolean From cd4e3e0f0af270100555a28cfb84f3fcdde3add3 Mon Sep 17 00:00:00 2001 From: "Reynaldo H. Verdejo Pinochet" Date: Fri, 15 Apr 2016 13:22:51 -0700 Subject: [PATCH 252/381] Drop usage of 'overlayed' to mean 'overlaid' --- gst/compositor/compositor.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gst/compositor/compositor.c b/gst/compositor/compositor.c index 1c0b39bcfe..07311b3741 100644 --- a/gst/compositor/compositor.c +++ b/gst/compositor/compositor.c @@ -75,7 +75,7 @@ * ]| A pipeline to demonstrate compositor used together with videobox. * This should show a 320x240 pixels video test source with some transparency * showing the background checker pattern. Another video test source with just - * the snow pattern of 100x100 pixels is overlayed on top of the first one on + * the snow pattern of 100x100 pixels is overlaid on top of the first one on * the left vertically centered with a small transparency showing the first * video test source behind and the checker pattern under it. Note that the * framerate of the output video is 10 frames per second. From cf6a36721bd9ab4a12eab664c99e4af7c27fc059 Mon Sep 17 00:00:00 2001 From: Thiago Santos Date: Mon, 28 Mar 2016 15:44:27 -0300 Subject: [PATCH 253/381] videoaggregator: properly handle interlace-mode restrictions videoaggregator can't handle interlace-mode changes so it must always restrict itself to the first interlacing mode it receives. Tests included https://bugzilla.gnome.org/show_bug.cgi?id=754495 --- gst-libs/gst/video/gstvideoaggregator.c | 59 +++++++-- tests/check/elements/compositor.c | 167 +++++++++++++++++++++++- 2 files changed, 217 insertions(+), 9 deletions(-) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index f9232e8dda..b7161ba182 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -819,6 +819,26 @@ done: return ret; } +static gboolean +gst_videoaggregator_get_sinkpads_interlace_mode (GstVideoAggregator * vagg, + GstVideoAggregatorPad * skip_pad, GstVideoInterlaceMode * mode) +{ + GList *walk; + + for (walk = GST_ELEMENT (vagg)->sinkpads; walk; walk = g_list_next (walk)) { + GstVideoAggregatorPad *vaggpad = walk->data; + + if (skip_pad && vaggpad == skip_pad) + continue; + if (vaggpad->info.finfo + && GST_VIDEO_INFO_FORMAT (&vaggpad->info) != GST_VIDEO_FORMAT_UNKNOWN) { + *mode = GST_VIDEO_INFO_INTERLACE_MODE (&vaggpad->info); + return TRUE; + } + } + return FALSE; +} + static gboolean gst_videoaggregator_pad_sink_setcaps (GstPad * pad, GstObject * parent, GstCaps * caps) @@ -839,14 +859,28 @@ gst_videoaggregator_pad_sink_setcaps (GstPad * pad, GstObject * parent, } GST_VIDEO_AGGREGATOR_LOCK (vagg); - if (GST_VIDEO_INFO_FORMAT (&vagg->info) != GST_VIDEO_FORMAT_UNKNOWN) { - if (GST_VIDEO_INFO_INTERLACE_MODE (&vagg->info) != - GST_VIDEO_INFO_INTERLACE_MODE (&info)) { - GST_ERROR_OBJECT (pad, - "got input caps %" GST_PTR_FORMAT ", but " "current caps are %" - GST_PTR_FORMAT, caps, vagg->priv->current_caps); - GST_VIDEO_AGGREGATOR_UNLOCK (vagg); - return FALSE; + { + GstVideoInterlaceMode pads_mode = GST_VIDEO_INTERLACE_MODE_PROGRESSIVE; + gboolean has_mode = FALSE; + + /* get the current output setting or fallback to other pads settings */ + if (GST_VIDEO_INFO_FORMAT (&vagg->info) != GST_VIDEO_FORMAT_UNKNOWN) { + pads_mode = GST_VIDEO_INFO_INTERLACE_MODE (&vagg->info); + has_mode = TRUE; + } else { + has_mode = + gst_videoaggregator_get_sinkpads_interlace_mode (vagg, vaggpad, + &pads_mode); + } + + if (has_mode) { + if (pads_mode != GST_VIDEO_INFO_INTERLACE_MODE (&info)) { + GST_ERROR_OBJECT (pad, + "got input caps %" GST_PTR_FORMAT ", but current caps are %" + GST_PTR_FORMAT, caps, vagg->priv->current_caps); + GST_VIDEO_AGGREGATOR_UNLOCK (vagg); + return FALSE; + } } } @@ -918,6 +952,8 @@ gst_videoaggregator_pad_sink_getcaps (GstPad * pad, GstVideoAggregator * vagg, GstAggregator *agg = GST_AGGREGATOR (vagg); GstPad *srcpad = GST_PAD (agg->srcpad); gboolean has_alpha; + GstVideoInterlaceMode interlace_mode; + gboolean has_interlace_mode; template_caps = gst_pad_get_pad_template_caps (srcpad); @@ -927,6 +963,10 @@ gst_videoaggregator_pad_sink_getcaps (GstPad * pad, GstVideoAggregator * vagg, srccaps = gst_caps_make_writable (srccaps); has_alpha = gst_videoaggregator_caps_has_alpha (srccaps); + has_interlace_mode = + gst_videoaggregator_get_sinkpads_interlace_mode (vagg, NULL, + &interlace_mode); + n = gst_caps_get_size (srccaps); for (i = 0; i < n; i++) { s = gst_caps_get_structure (srccaps, i); @@ -936,6 +976,9 @@ gst_videoaggregator_pad_sink_getcaps (GstPad * pad, GstVideoAggregator * vagg, gst_structure_remove_fields (s, "colorimetry", "chroma-site", "format", "pixel-aspect-ratio", NULL); + if (has_interlace_mode) + gst_structure_set (s, "interlace-mode", G_TYPE_STRING, + gst_video_interlace_mode_to_string (interlace_mode), NULL); } if (filter) { diff --git a/tests/check/elements/compositor.c b/tests/check/elements/compositor.c index 6e7fe22d73..258d1d96cd 100644 --- a/tests/check/elements/compositor.c +++ b/tests/check/elements/compositor.c @@ -355,6 +355,15 @@ GST_START_TEST (test_caps_query) gst_caps_unref (caps); gst_caps_unref (restriction_caps); + /* check that compositor proxies downstream interlaced-mode */ + restriction_caps = + gst_caps_from_string ("video/x-raw, interlace-mode=(string)interleaved"); + g_object_set (capsfilter, "caps", restriction_caps, NULL); + caps = gst_pad_query_caps (sinkpad, NULL); + fail_unless (gst_caps_is_subset (caps, restriction_caps)); + gst_caps_unref (caps); + gst_caps_unref (restriction_caps); + gst_element_set_state (pipeline, GST_STATE_NULL); gst_element_release_request_pad (compositor, sinkpad); gst_object_unref (sinkpad); @@ -365,6 +374,82 @@ GST_START_TEST (test_caps_query) GST_END_TEST; + +GST_START_TEST (test_caps_query_interlaced) +{ + GstElement *compositor, *sink; + GstElement *pipeline; + gboolean res; + GstStateChangeReturn state_res; + GstPad *sinkpad; + GstCaps *caps; + GstCaps *caps_mixed, *caps_progressive, *caps_interleaved; + GstEvent *caps_event; + + caps_interleaved = + gst_caps_from_string ("video/x-raw, interlace-mode=interleaved"); + caps_mixed = gst_caps_from_string ("video/x-raw, interlace-mode=mixed"); + caps_progressive = + gst_caps_from_string ("video/x-raw, interlace-mode=progressive"); + + /* initial setup */ + compositor = gst_element_factory_make ("compositor", "compositor"); + sink = gst_element_factory_make ("fakesink", "sink"); + pipeline = gst_pipeline_new ("test-pipeline"); + + gst_bin_add_many (GST_BIN (pipeline), compositor, sink, NULL); + res = gst_element_link (compositor, sink); + fail_unless (res == TRUE, NULL); + sinkpad = gst_element_get_request_pad (compositor, "sink_%u"); + + state_res = gst_element_set_state (pipeline, GST_STATE_PLAYING); + fail_if (state_res == GST_STATE_CHANGE_FAILURE); + + /* try an unrestricted caps query, should be compatible with all formats */ + caps = gst_pad_query_caps (sinkpad, NULL); + fail_unless (gst_caps_can_intersect (caps, caps_interleaved)); + fail_unless (gst_caps_can_intersect (caps, caps_progressive)); + fail_unless (gst_caps_can_intersect (caps, caps_mixed)); + gst_caps_unref (caps); + + /* now set caps on the pad, it should restrict the interlaced-mode for + * future caps */ + caps = gst_caps_from_string ("video/x-raw, width=100, height=100, " + "format=RGB, framerate=1/1, interlace-mode=progressive"); + caps_event = gst_event_new_caps (caps); + gst_caps_unref (caps); + fail_unless (gst_pad_send_event (sinkpad, caps_event)); + + /* now recheck the interlace-mode */ + gst_object_unref (sinkpad); + sinkpad = gst_element_get_request_pad (compositor, "sink_%u"); + caps = gst_pad_query_caps (sinkpad, NULL); + fail_if (gst_caps_can_intersect (caps, caps_interleaved)); + fail_unless (gst_caps_can_intersect (caps, caps_progressive)); + fail_if (gst_caps_can_intersect (caps, caps_mixed)); + gst_object_unref (sinkpad); + gst_caps_unref (caps); + + gst_element_set_state (pipeline, GST_STATE_NULL); + gst_object_unref (pipeline); + gst_caps_unref (caps_interleaved); + gst_caps_unref (caps_mixed); + gst_caps_unref (caps_progressive); +} + +GST_END_TEST; + +static void +add_interlaced_mode_to_caps (GstCaps * caps, const gchar * mode) +{ + GstStructure *s; + + for (gint i = 0; i < gst_caps_get_size (caps); i++) { + s = gst_caps_get_structure (caps, i); + gst_structure_set (s, "interlace-mode", G_TYPE_STRING, mode, NULL); + } +} + #define MODE_ALL 1 #define MODE_NON_ALPHA 2 @@ -384,6 +469,11 @@ run_late_caps_query_test (GstCaps * input_caps, GstCaps * output_allowed_caps, all_caps = _compositor_get_all_supported_caps (); non_alpha_caps = _compositor_get_non_alpha_supported_caps (); + /* add progressive mode as it is what is used in the test, otherwise + * is_equal checks would fail */ + add_interlaced_mode_to_caps (all_caps, "progressive"); + add_interlaced_mode_to_caps (non_alpha_caps, "progressive"); + compositor = gst_element_factory_make ("compositor", "compositor"); capsfilter = gst_element_factory_make ("capsfilter", "out-cf"); sink = gst_element_factory_make ("fakesink", "sink"); @@ -450,7 +540,6 @@ GST_START_TEST (test_late_caps_query) rgb_caps = gst_caps_from_string ("video/x-raw, format=(string)RGB, " "width=(int)100, height=(int)100, framerate=(fraction)1/1"); - non_alpha_caps = gst_caps_from_string ("video/x-raw, format=(string)RGB"); /* check that a 2nd pad that is added late to compositor will be able to @@ -465,6 +554,80 @@ GST_START_TEST (test_late_caps_query) GST_END_TEST; +static void +run_late_caps_set_test (GstCaps * first_caps, GstCaps * expected_query_caps, + GstCaps * second_caps, gboolean accept_caps) +{ + GstElement *capsfilter_1; + GstElement *compositor; + GstElement *pipeline; + GstStateChangeReturn state_res; + GstPad *sinkpad_2; + GstCaps *caps; + GstEvent *caps_event; + GstBus *bus; + GstMessage *msg; + + pipeline = + gst_parse_launch ("videotestsrc num-buffers=10 ! capsfilter name=cf1 !" + " compositor name=c ! fakesink sync=true", NULL); + fail_unless (pipeline != NULL); + + bus = gst_pipeline_get_bus (GST_PIPELINE (pipeline)); + + compositor = gst_bin_get_by_name (GST_BIN (pipeline), "c"); + capsfilter_1 = gst_bin_get_by_name (GST_BIN (pipeline), "cf1"); + + g_object_set (capsfilter_1, "caps", first_caps, NULL); + + state_res = gst_element_set_state (pipeline, GST_STATE_PAUSED); + fail_if (state_res == GST_STATE_CHANGE_FAILURE); + + /* wait for pipeline to get to paused */ + msg = + gst_bus_timed_pop_filtered (bus, GST_CLOCK_TIME_NONE, + GST_MESSAGE_ASYNC_DONE); + fail_unless (msg != NULL); + fail_unless (GST_MESSAGE_TYPE (msg) == GST_MESSAGE_ASYNC_DONE); + gst_message_unref (msg); + + /* try to set the second caps */ + sinkpad_2 = gst_element_get_request_pad (compositor, "sink_%u"); + caps = gst_pad_query_caps (sinkpad_2, NULL); + fail_unless (gst_caps_is_subset (expected_query_caps, caps)); + caps_event = gst_event_new_caps (second_caps); + fail_unless (gst_pad_send_event (sinkpad_2, caps_event) == accept_caps); + gst_caps_unref (caps); + gst_object_unref (sinkpad_2); + + gst_object_unref (bus); + gst_object_unref (compositor); + gst_object_unref (capsfilter_1); + gst_element_set_state (pipeline, GST_STATE_NULL); + gst_object_unref (pipeline); +} + +GST_START_TEST (test_late_caps_different_interlaced) +{ + GstCaps *non_interlaced_caps; + GstCaps *interlaced_caps; + + non_interlaced_caps = + gst_caps_from_string ("video/x-raw, interlace-mode=progressive, " + "format=RGB, width=100, height=100, framerate=1/1"); + interlaced_caps = + gst_caps_from_string ("video/x-raw, interlace-mode=interleaved, " + "format=RGB, width=100, height=100, framerate=1/1"); + + run_late_caps_set_test (non_interlaced_caps, non_interlaced_caps, + interlaced_caps, FALSE); + + gst_caps_unref (non_interlaced_caps); + gst_caps_unref (interlaced_caps); +} + +GST_END_TEST; + static guint play_count = 0; static GstEvent *play_seek_event = NULL; @@ -1915,7 +2078,9 @@ compositor_suite (void) tcase_add_test (tc_chain, test_caps); tcase_add_test (tc_chain, test_event); tcase_add_test (tc_chain, test_caps_query); + tcase_add_test (tc_chain, test_caps_query_interlaced); tcase_add_test (tc_chain, test_late_caps_query); + tcase_add_test (tc_chain, test_late_caps_different_interlaced); tcase_add_test (tc_chain, test_play_twice); tcase_add_test (tc_chain, test_play_twice_then_add_and_play_again); tcase_add_test (tc_chain, test_add_pad); From 14c9e37fddb17e273935e2dce4a7b7a87c4f8655 Mon Sep 17 00:00:00 2001 From: Thiago Santos Date: Mon, 18 Apr 2016 13:46:55 -0300 Subject: [PATCH 254/381] videoaggregator: plug caps leak It was losing ref of the original 'ret' caps that would be returned or returning it with 2 references to it. --- gst-libs/gst/video/gstvideoaggregator.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index b7161ba182..11ed018202 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -680,9 +680,10 @@ gst_videoaggregator_default_update_caps (GstVideoAggregator * vagg, ret = gst_caps_merge (best_format_caps, gst_caps_ref (caps)); if (filter) { - ret = gst_caps_intersect (ret, filter); - } else { - gst_caps_ref (ret); + GstCaps *tmp; + tmp = gst_caps_intersect (ret, filter); + gst_caps_unref (ret); + ret = tmp; } return ret; From 905c1600b11a2e1386ae1c62e3a12e0d7ebf8a0d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Tue, 3 May 2016 11:11:24 +0300 Subject: [PATCH 255/381] compositor: Set blend functions in ::negotiated_caps() instead of ::fixate_caps() The latter should not change any state but just fixate the caps, while the former is always called when srcpads caps are decided. https://bugzilla.gnome.org/show_bug.cgi?id=765324 --- gst/compositor/compositor.c | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/gst/compositor/compositor.c b/gst/compositor/compositor.c index 07311b3741..100316bb6f 100644 --- a/gst/compositor/compositor.c +++ b/gst/compositor/compositor.c @@ -954,17 +954,25 @@ _fixate_caps (GstVideoAggregator * vagg, GstCaps * caps) best_fps_d); ret = gst_caps_fixate (ret); - if (best_width > 0 && best_height > 0) { - GstVideoInfo v_info; + return ret; +} - gst_video_info_from_caps (&v_info, ret); - if (!set_functions (GST_COMPOSITOR (vagg), &v_info)) { - GST_ERROR_OBJECT (vagg, "Failed to setup vfuncs"); - return NULL; - } +static gboolean +_negotiated_caps (GstVideoAggregator * vagg, GstCaps * caps) +{ + GstVideoInfo v_info; + + GST_DEBUG_OBJECT (vagg, "Negotiated caps %" GST_PTR_FORMAT, caps); + + if (!gst_video_info_from_caps (&v_info, caps)) + return FALSE; + + if (!set_functions (GST_COMPOSITOR (vagg), &v_info)) { + GST_ERROR_OBJECT (vagg, "Failed to setup vfuncs"); + return FALSE; } - return ret; + return TRUE; } static GstFlowReturn @@ -1096,6 +1104,7 @@ gst_compositor_class_init (GstCompositorClass * klass) agg_class->sinkpads_type = GST_TYPE_COMPOSITOR_PAD; agg_class->sink_query = _sink_query; videoaggregator_class->fixate_caps = _fixate_caps; + videoaggregator_class->negotiated_caps = _negotiated_caps; videoaggregator_class->aggregate_frames = gst_compositor_aggregate_frames; g_object_class_install_property (gobject_class, PROP_BACKGROUND, From 0283c3d8aa764c43dae7de6675d7fe57154218c8 Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Wed, 4 May 2016 12:17:59 +1000 Subject: [PATCH 256/381] gl/egl: replace gsteglimagememory with an EGLImage wrapper That can be passed to GstGLMemoryEGL. This also ports the dmabuf uploader to GstEGLImage and GstGLMemoryEGL. --- ext/gl/gstglmixer.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/ext/gl/gstglmixer.c b/ext/gl/gstglmixer.c index 3fc7fce2c8..764a53c33e 100644 --- a/ext/gl/gstglmixer.c +++ b/ext/gl/gstglmixer.c @@ -28,10 +28,6 @@ #include "gstglmixer.h" -#if GST_GL_HAVE_PLATFORM_EGL -#include -#endif - #define gst_gl_mixer_parent_class parent_class G_DEFINE_ABSTRACT_TYPE (GstGLMixer, gst_gl_mixer, GST_TYPE_GL_BASE_MIXER); From 96682c41d4e59f2f06655ee0341a8369d50ffeb9 Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Sat, 14 May 2016 15:50:57 +0300 Subject: [PATCH 257/381] glbasemixer: actually attempt to propose an allocation upstream We were always failing the allocation query as a flag was never being set to signal a successful negotiation. Fix by setting the required flag on a successful caps event from upstream. --- ext/gl/gstglbasemixer.c | 249 ++++++++++++++++++++++++---------------- ext/gl/gstglbasemixer.h | 2 + 2 files changed, 151 insertions(+), 100 deletions(-) diff --git a/ext/gl/gstglbasemixer.c b/ext/gl/gstglbasemixer.c index c78ea8889a..9d2851e33d 100644 --- a/ext/gl/gstglbasemixer.c +++ b/ext/gl/gstglbasemixer.c @@ -122,6 +122,131 @@ _default_propose_allocation (GstGLBaseMixer * mix, GstGLBaseMixerPad * pad, return TRUE; } +static gboolean +gst_gl_base_mixer_sink_event (GstAggregator * agg, GstAggregatorPad * bpad, + GstEvent * event) +{ + GstGLBaseMixerPad *pad = GST_GL_BASE_MIXER_PAD (bpad); + + switch (GST_EVENT_TYPE (event)) { + case GST_EVENT_CAPS: + if (!GST_AGGREGATOR_CLASS (parent_class)->sink_event (agg, bpad, event)) + return FALSE; + + pad->negotiated = TRUE; + return TRUE; + default: + break; + } + + return GST_AGGREGATOR_CLASS (parent_class)->sink_event (agg, bpad, event); +} + +static gboolean +_find_local_gl_context (GstGLBaseMixer * mix) +{ + GstQuery *query; + GstContext *context; + const GstStructure *s; + + if (mix->context) + return TRUE; + + query = gst_query_new_context ("gst.gl.local_context"); + if (!mix->context && gst_gl_run_query (GST_ELEMENT (mix), query, GST_PAD_SRC)) { + gst_query_parse_context (query, &context); + if (context) { + s = gst_context_get_structure (context); + gst_structure_get (s, "context", GST_GL_TYPE_CONTEXT, &mix->context, + NULL); + } + } + if (!mix->context + && gst_gl_run_query (GST_ELEMENT (mix), query, GST_PAD_SINK)) { + gst_query_parse_context (query, &context); + if (context) { + s = gst_context_get_structure (context); + gst_structure_get (s, "context", GST_GL_TYPE_CONTEXT, &mix->context, + NULL); + } + } + + GST_DEBUG_OBJECT (mix, "found local context %p", mix->context); + + gst_query_unref (query); + + if (mix->context) + return TRUE; + + return FALSE; +} + +static gboolean +_get_gl_context (GstGLBaseMixer * mix) +{ + GstGLBaseMixerClass *mix_class = GST_GL_BASE_MIXER_GET_CLASS (mix); + GError *error = NULL; + + if (!gst_gl_ensure_element_data (mix, &mix->display, + &mix->priv->other_context)) + return FALSE; + + gst_gl_display_filter_gl_api (mix->display, mix_class->supported_gl_api); + + _find_local_gl_context (mix); + + if (!mix->context) { + GST_OBJECT_LOCK (mix->display); + do { + if (mix->context) { + gst_object_unref (mix->context); + mix->context = NULL; + } + /* just get a GL context. we don't care */ + mix->context = + gst_gl_display_get_gl_context_for_thread (mix->display, NULL); + if (!mix->context) { + if (!gst_gl_display_create_context (mix->display, + mix->priv->other_context, &mix->context, &error)) { + GST_OBJECT_UNLOCK (mix->display); + goto context_error; + } + } + } while (!gst_gl_display_add_context (mix->display, mix->context)); + GST_OBJECT_UNLOCK (mix->display); + } + + { + GstGLAPI current_gl_api = gst_gl_context_get_gl_api (mix->context); + if ((current_gl_api & mix_class->supported_gl_api) == 0) + goto unsupported_gl_api; + } + + return TRUE; + +unsupported_gl_api: + { + GstGLAPI gl_api = gst_gl_context_get_gl_api (mix->context); + gchar *gl_api_str = gst_gl_api_to_string (gl_api); + gchar *supported_gl_api_str = + gst_gl_api_to_string (mix_class->supported_gl_api); + GST_ELEMENT_ERROR (mix, RESOURCE, BUSY, + ("GL API's not compatible context: %s supported: %s", gl_api_str, + supported_gl_api_str), (NULL)); + + g_free (supported_gl_api_str); + g_free (gl_api_str); + return FALSE; + } +context_error: + { + GST_ELEMENT_ERROR (mix, RESOURCE, NOT_FOUND, ("%s", error->message), + (NULL)); + g_clear_error (&error); + return FALSE; + } +} + static gboolean gst_gl_base_mixer_sink_query (GstAggregator * agg, GstAggregatorPad * bpad, GstQuery * query) @@ -139,16 +264,20 @@ gst_gl_base_mixer_sink_query (GstAggregator * agg, GstAggregatorPad * bpad, GstQuery *decide_query = NULL; GST_OBJECT_LOCK (mix); - if (G_UNLIKELY (!mix->priv->negotiated)) { + if (G_UNLIKELY (!pad->negotiated)) { GST_DEBUG_OBJECT (mix, "not negotiated yet, can't answer ALLOCATION query"); GST_OBJECT_UNLOCK (mix); return FALSE; } + if ((decide_query = mix->priv->query)) gst_query_ref (decide_query); GST_OBJECT_UNLOCK (mix); + if (!_get_gl_context (mix)) + return FALSE; + GST_DEBUG_OBJECT (mix, "calling propose allocation with query %" GST_PTR_FORMAT, decide_query); @@ -273,6 +402,7 @@ gst_gl_base_mixer_class_init (GstGLBaseMixerClass * klass) agg_class->sinkpads_type = GST_TYPE_GL_BASE_MIXER_PAD; agg_class->sink_query = gst_gl_base_mixer_sink_query; + agg_class->sink_event = gst_gl_base_mixer_sink_event; agg_class->src_query = gst_gl_base_mixer_src_query; agg_class->src_activate = gst_gl_base_mixer_src_activate_mode; agg_class->stop = gst_gl_base_mixer_stop; @@ -294,11 +424,24 @@ gst_gl_base_mixer_class_init (GstGLBaseMixerClass * klass) klass->supported_gl_api = GST_GL_API_ANY; } +static gboolean +_reset_pad (GstAggregator * self, GstAggregatorPad * base_pad, + gpointer user_data) +{ + GstGLBaseMixerPad *mix_pad = GST_GL_BASE_MIXER_PAD (base_pad); + + mix_pad->negotiated = FALSE; + + return TRUE; +} + static void gst_gl_base_mixer_reset (GstGLBaseMixer * mix) { /* clean up collect data */ - mix->priv->negotiated = FALSE; + + gst_aggregator_iterate_sinkpads (GST_AGGREGATOR (mix), + (GstAggregatorPadForeachFunc) _reset_pad, NULL); } static void @@ -414,113 +557,19 @@ gst_gl_base_mixer_src_query (GstAggregator * agg, GstQuery * query) return GST_AGGREGATOR_CLASS (parent_class)->src_query (agg, query); } -static gboolean -_find_local_gl_context (GstGLBaseMixer * mix) -{ - GstQuery *query; - GstContext *context; - const GstStructure *s; - - if (mix->context) - return TRUE; - - query = gst_query_new_context ("gst.gl.local_context"); - if (!mix->context && gst_gl_run_query (GST_ELEMENT (mix), query, GST_PAD_SRC)) { - gst_query_parse_context (query, &context); - if (context) { - s = gst_context_get_structure (context); - gst_structure_get (s, "context", GST_GL_TYPE_CONTEXT, &mix->context, - NULL); - } - } - if (!mix->context - && gst_gl_run_query (GST_ELEMENT (mix), query, GST_PAD_SINK)) { - gst_query_parse_context (query, &context); - if (context) { - s = gst_context_get_structure (context); - gst_structure_get (s, "context", GST_GL_TYPE_CONTEXT, &mix->context, - NULL); - } - } - - GST_DEBUG_OBJECT (mix, "found local context %p", mix->context); - - gst_query_unref (query); - - if (mix->context) - return TRUE; - - return FALSE; -} - static gboolean gst_gl_base_mixer_decide_allocation (GstGLBaseMixer * mix, GstQuery * query) { GstGLBaseMixerClass *mix_class = GST_GL_BASE_MIXER_GET_CLASS (mix); - GError *error = NULL; - gboolean ret = TRUE; - if (!gst_gl_ensure_element_data (mix, &mix->display, - &mix->priv->other_context)) + if (!_get_gl_context (mix)) return FALSE; - gst_gl_display_filter_gl_api (mix->display, mix_class->supported_gl_api); - - _find_local_gl_context (mix); - - if (!mix->context) { - GST_OBJECT_LOCK (mix->display); - do { - if (mix->context) { - gst_object_unref (mix->context); - mix->context = NULL; - } - /* just get a GL context. we don't care */ - mix->context = - gst_gl_display_get_gl_context_for_thread (mix->display, NULL); - if (!mix->context) { - if (!gst_gl_display_create_context (mix->display, - mix->priv->other_context, &mix->context, &error)) { - GST_OBJECT_UNLOCK (mix->display); - goto context_error; - } - } - } while (!gst_gl_display_add_context (mix->display, mix->context)); - GST_OBJECT_UNLOCK (mix->display); - } - - { - GstGLAPI current_gl_api = gst_gl_context_get_gl_api (mix->context); - if ((current_gl_api & mix_class->supported_gl_api) == 0) - goto unsupported_gl_api; - } - if (mix_class->decide_allocation) - ret = mix_class->decide_allocation (mix, query); + if (!mix_class->decide_allocation (mix, query)) + return FALSE; - return ret; - -unsupported_gl_api: - { - GstGLAPI gl_api = gst_gl_context_get_gl_api (mix->context); - gchar *gl_api_str = gst_gl_api_to_string (gl_api); - gchar *supported_gl_api_str = - gst_gl_api_to_string (mix_class->supported_gl_api); - GST_ELEMENT_ERROR (mix, RESOURCE, BUSY, - ("GL API's not compatible context: %s supported: %s", gl_api_str, - supported_gl_api_str), (NULL)); - - g_free (supported_gl_api_str); - g_free (gl_api_str); - return FALSE; - } -context_error: - { - GST_ELEMENT_ERROR (mix, RESOURCE, NOT_FOUND, ("%s", error->message), - (NULL)); - g_clear_error (&error); - return FALSE; - } + return TRUE; } /* takes ownership of the pool, allocator and query */ diff --git a/ext/gl/gstglbasemixer.h b/ext/gl/gstglbasemixer.h index 6282ccb96e..cba8184792 100644 --- a/ext/gl/gstglbasemixer.h +++ b/ext/gl/gstglbasemixer.h @@ -47,6 +47,8 @@ typedef struct _GstGLBaseMixerPadClass GstGLBaseMixerPadClass; struct _GstGLBaseMixerPad { GstVideoAggregatorPad parent; /* subclass the pad */ + + gboolean negotiated; }; struct _GstGLBaseMixerPadClass From 06986222c926b5f7a7b5b4536cd8ccaadc3b66a7 Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Sat, 14 May 2016 16:27:26 +0300 Subject: [PATCH 258/381] gl: take the affine transformation in NDC Provide a function to get the affine matrix in the meta in terms of NDC coordinates and use as a standard opengl matrix. Also advertise support for the affine transformation meta in the allocation query. --- ext/gl/gstglvideomixer.c | 40 ++++++++++++++++++++++++++-------------- 1 file changed, 26 insertions(+), 14 deletions(-) diff --git a/ext/gl/gstglvideomixer.c b/ext/gl/gstglvideomixer.c index ba3caa4fd1..83905d0e68 100644 --- a/ext/gl/gstglvideomixer.c +++ b/ext/gl/gstglvideomixer.c @@ -454,6 +454,7 @@ enum #define DEBUG_INIT \ GST_DEBUG_CATEGORY_INIT (gst_gl_video_mixer_debug, "glvideomixer", 0, "glvideomixer element"); +#define gst_gl_video_mixer_parent_class parent_class G_DEFINE_TYPE_WITH_CODE (GstGLVideoMixer, gst_gl_video_mixer, GST_TYPE_GL_MIXER, DEBUG_INIT); @@ -465,6 +466,9 @@ static void gst_gl_video_mixer_get_property (GObject * object, guint prop_id, static GstCaps *_update_caps (GstVideoAggregator * vagg, GstCaps * caps, GstCaps * filter); static GstCaps *_fixate_caps (GstVideoAggregator * vagg, GstCaps * caps); +static gboolean gst_gl_video_mixer_propose_allocation (GstGLBaseMixer * + base_mix, GstGLBaseMixerPad * base_pad, GstQuery * decide_query, + GstQuery * query); static void gst_gl_video_mixer_reset (GstGLMixer * mixer); static gboolean gst_gl_video_mixer_init_shader (GstGLMixer * mixer, GstCaps * outcaps); @@ -856,6 +860,7 @@ gst_gl_video_mixer_class_init (GstGLVideoMixerClass * klass) GstElementClass *element_class; GstAggregatorClass *agg_class = (GstAggregatorClass *) klass; GstVideoAggregatorClass *vagg_class = (GstVideoAggregatorClass *) klass; + GstGLBaseMixerClass *mix_class = GST_GL_BASE_MIXER_CLASS (klass); gobject_class = (GObjectClass *) klass; element_class = GST_ELEMENT_CLASS (klass); @@ -883,6 +888,8 @@ gst_gl_video_mixer_class_init (GstGLVideoMixerClass * klass) agg_class->sinkpads_type = GST_TYPE_GL_VIDEO_MIXER_PAD; + mix_class->propose_allocation = gst_gl_video_mixer_propose_allocation; + GST_GL_BASE_MIXER_CLASS (klass)->supported_gl_api = GST_GL_API_OPENGL | GST_GL_API_OPENGL3 | GST_GL_API_GLES2; } @@ -926,6 +933,20 @@ gst_gl_video_mixer_get_property (GObject * object, guint prop_id, } } +static gboolean +gst_gl_video_mixer_propose_allocation (GstGLBaseMixer * base_mix, + GstGLBaseMixerPad * base_pad, GstQuery * decide_query, GstQuery * query) +{ + if (!GST_GL_BASE_MIXER_CLASS (parent_class)->propose_allocation (base_mix, + base_pad, decide_query, query)) + return FALSE; + + gst_query_add_allocation_meta (query, + GST_VIDEO_AFFINE_TRANSFORMATION_META_API_TYPE, 0); + + return TRUE; +} + static void _mixer_pad_get_output_size (GstGLVideoMixer * mix, GstGLVideoMixerPad * mix_pad, gint out_par_n, gint out_par_d, gint * width, @@ -1130,7 +1151,7 @@ gst_gl_video_mixer_init_shader (GstGLMixer * mixer, GstCaps * outcaps) video_mixer->shader); return gst_gl_context_gen_shader (GST_GL_BASE_MIXER (mixer)->context, - gst_gl_shader_string_vertex_mat4_texture_transform, + gst_gl_shader_string_vertex_mat4_vertex_transform, video_mixer_f_src, &video_mixer->shader); } @@ -1361,13 +1382,6 @@ _set_blend_state (GstGLVideoMixer * video_mixer, GstGLVideoMixerPad * mix_pad) return TRUE; } -static const gfloat identity_matrix[] = { - 1.0f, 0.0f, 0.0f, 0.0f, - 0.0f, 1.0f, 0.0f, 0.0f, - 0.0f, 0.0f, 1.0f, 0.0f, - 0.0f, 0.0f, 0.0f, 1.0f, -}; - /* opengl scene, params: input texture (not the output mixer->texture) */ static void gst_gl_video_mixer_callback (gpointer stuff) @@ -1495,15 +1509,13 @@ gst_gl_video_mixer_callback (gpointer stuff) { GstVideoAffineTransformationMeta *af_meta; + gfloat matrix[16]; af_meta = gst_buffer_get_video_affine_transformation_meta (vagg_pad->buffer); - if (af_meta) - gst_gl_shader_set_uniform_matrix_4fv (video_mixer->shader, - "u_transformation", 1, FALSE, af_meta->matrix); - else - gst_gl_shader_set_uniform_matrix_4fv (video_mixer->shader, - "u_transformation", 1, FALSE, identity_matrix); + gst_gl_get_affine_transformation_meta_as_ndc (af_meta, matrix); + gst_gl_shader_set_uniform_matrix_4fv (video_mixer->shader, + "u_transformation", 1, FALSE, matrix); } gl->EnableVertexAttribArray (attr_position_loc); From d44a77a5e89a2e7ab4132ba9ce8171037edd9c0c Mon Sep 17 00:00:00 2001 From: Edward Hervey Date: Sun, 15 May 2016 14:34:33 +0200 Subject: [PATCH 259/381] compositor: Check if we get a valid display ratio As is done everywhere else, and avoids setting bogus values And remove useless * checks (we always provide valid values and it's an internal function). CID #1320700 --- gst/compositor/compositor.c | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/gst/compositor/compositor.c b/gst/compositor/compositor.c index 100316bb6f..c9fd3fc1ea 100644 --- a/gst/compositor/compositor.c +++ b/gst/compositor/compositor.c @@ -239,9 +239,12 @@ _mixer_pad_get_output_size (GstCompositor * comp, comp_pad->height <= 0 ? GST_VIDEO_INFO_HEIGHT (&vagg_pad->info) : comp_pad->height; - gst_video_calculate_display_ratio (&dar_n, &dar_d, pad_width, pad_height, - GST_VIDEO_INFO_PAR_N (&vagg_pad->info), - GST_VIDEO_INFO_PAR_D (&vagg_pad->info), out_par_n, out_par_d); + if (!gst_video_calculate_display_ratio (&dar_n, &dar_d, pad_width, pad_height, + GST_VIDEO_INFO_PAR_N (&vagg_pad->info), + GST_VIDEO_INFO_PAR_D (&vagg_pad->info), out_par_n, out_par_d)) { + GST_WARNING_OBJECT (comp_pad, "Cannot calculate display aspect ratio"); + *width = *height = 0; + } GST_LOG_OBJECT (comp_pad, "scaling %ux%u by %u/%u (%u/%u / %u/%u)", pad_width, pad_height, dar_n, dar_d, GST_VIDEO_INFO_PAR_N (&vagg_pad->info), GST_VIDEO_INFO_PAR_D (&vagg_pad->info), out_par_n, out_par_d); @@ -254,10 +257,8 @@ _mixer_pad_get_output_size (GstCompositor * comp, pad_width = gst_util_uint64_scale_int (pad_height, dar_n, dar_d); } - if (width) - *width = pad_width; - if (height) - *height = pad_height; + *width = pad_width; + *height = pad_height; } static gboolean From b89a47ed6686e5e5fcee9bd60b43d886dd88becc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim-Philipp=20M=C3=BCller?= Date: Mon, 16 May 2016 12:27:50 +0100 Subject: [PATCH 260/381] videoaggregator: canonicalise function names Had to be done at some point: gst_videoaggregator_* -> gst_video_aggregator_* Also fix up some function names with typos. --- gst-libs/gst/video/gstvideoaggregator.c | 240 ++++++++++++------------ gst-libs/gst/video/gstvideoaggregator.h | 4 +- 2 files changed, 124 insertions(+), 120 deletions(-) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index 11ed018202..49ab5ae84d 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -43,11 +43,11 @@ #include "gstvideoaggregator.h" #include "gstvideoaggregatorpad.h" -GST_DEBUG_CATEGORY_STATIC (gst_videoaggregator_debug); -#define GST_CAT_DEFAULT gst_videoaggregator_debug +GST_DEBUG_CATEGORY_STATIC (gst_video_aggregator_debug); +#define GST_CAT_DEFAULT gst_video_aggregator_debug /* Needed prototypes */ -static void gst_videoaggregator_reset_qos (GstVideoAggregator * vagg); +static void gst_video_aggregator_reset_qos (GstVideoAggregator * vagg); /**************************************** * GstVideoAggregatorPad implementation * @@ -77,11 +77,11 @@ struct _GstVideoAggregatorPadPrivate }; -G_DEFINE_TYPE (GstVideoAggregatorPad, gst_videoaggregator_pad, +G_DEFINE_TYPE (GstVideoAggregatorPad, gst_video_aggregator_pad, GST_TYPE_AGGREGATOR_PAD); static void -gst_videoaggregator_pad_get_property (GObject * object, guint prop_id, +gst_video_aggregator_pad_get_property (GObject * object, guint prop_id, GValue * value, GParamSpec * pspec) { GstVideoAggregatorPad *pad = GST_VIDEO_AGGREGATOR_PAD (object); @@ -107,7 +107,7 @@ pad_zorder_compare (const GstVideoAggregatorPad * pad1, } static void -gst_videoaggregator_pad_set_property (GObject * object, guint prop_id, +gst_video_aggregator_pad_set_property (GObject * object, guint prop_id, const GValue * value, GParamSpec * pspec) { GstVideoAggregatorPad *pad = GST_VIDEO_AGGREGATOR_PAD (object); @@ -139,7 +139,7 @@ _flush_pad (GstAggregatorPad * aggpad, GstAggregator * aggregator) GstVideoAggregator *vagg = GST_VIDEO_AGGREGATOR (aggregator); GstVideoAggregatorPad *pad = GST_VIDEO_AGGREGATOR_PAD (aggpad); - gst_videoaggregator_reset_qos (vagg); + gst_video_aggregator_reset_qos (vagg); gst_buffer_replace (&pad->buffer, NULL); pad->priv->start_time = -1; pad->priv->end_time = -1; @@ -217,7 +217,7 @@ gst_video_aggregator_pad_set_info (GstVideoAggregatorPad * pad, } static void -gst_videoaggregator_pad_finalize (GObject * o) +gst_video_aggregator_pad_finalize (GObject * o) { GstVideoAggregatorPad *vaggpad = GST_VIDEO_AGGREGATOR_PAD (o); @@ -225,7 +225,7 @@ gst_videoaggregator_pad_finalize (GObject * o) gst_video_converter_free (vaggpad->priv->convert); vaggpad->priv->convert = NULL; - G_OBJECT_CLASS (gst_videoaggregator_pad_parent_class)->finalize (o); + G_OBJECT_CLASS (gst_video_aggregator_pad_parent_class)->finalize (o); } static gboolean @@ -300,14 +300,14 @@ gst_video_aggregator_pad_clean_frame (GstVideoAggregatorPad * pad, } static void -gst_videoaggregator_pad_class_init (GstVideoAggregatorPadClass * klass) +gst_video_aggregator_pad_class_init (GstVideoAggregatorPadClass * klass) { GObjectClass *gobject_class = (GObjectClass *) klass; GstAggregatorPadClass *aggpadclass = (GstAggregatorPadClass *) klass; - gobject_class->set_property = gst_videoaggregator_pad_set_property; - gobject_class->get_property = gst_videoaggregator_pad_get_property; - gobject_class->finalize = gst_videoaggregator_pad_finalize; + gobject_class->set_property = gst_video_aggregator_pad_set_property; + gobject_class->get_property = gst_video_aggregator_pad_get_property; + gobject_class->finalize = gst_video_aggregator_pad_finalize; g_object_class_install_property (gobject_class, PROP_PAD_ZORDER, g_param_spec_uint ("zorder", "Z-Order", "Z Order of the picture", @@ -329,7 +329,7 @@ gst_videoaggregator_pad_class_init (GstVideoAggregatorPadClass * klass) } static void -gst_videoaggregator_pad_init (GstVideoAggregatorPad * vaggpad) +gst_video_aggregator_pad_init (GstVideoAggregatorPad * vaggpad) { vaggpad->priv = G_TYPE_INSTANCE_GET_PRIVATE (vaggpad, GST_TYPE_VIDEO_AGGREGATOR_PAD, @@ -347,8 +347,8 @@ gst_videoaggregator_pad_init (GstVideoAggregatorPad * vaggpad) * GstChildProxy implementation * *********************************/ static GObject * -gst_videoaggregator_child_proxy_get_child_by_index (GstChildProxy * child_proxy, - guint index) +gst_video_aggregator_child_proxy_get_child_by_index (GstChildProxy * + child_proxy, guint index) { GObject *obj; GstVideoAggregator *vagg = GST_VIDEO_AGGREGATOR (child_proxy); @@ -362,7 +362,8 @@ gst_videoaggregator_child_proxy_get_child_by_index (GstChildProxy * child_proxy, } static guint -gst_videoaggregator_child_proxy_get_children_count (GstChildProxy * child_proxy) +gst_video_aggregator_child_proxy_get_children_count (GstChildProxy * + child_proxy) { guint count = 0; @@ -376,15 +377,15 @@ gst_videoaggregator_child_proxy_get_children_count (GstChildProxy * child_proxy) } static void -gst_videoaggregator_child_proxy_init (gpointer g_iface, gpointer iface_data) +gst_video_aggregator_child_proxy_init (gpointer g_iface, gpointer iface_data) { GstChildProxyInterface *iface = g_iface; GST_INFO ("intializing child proxy interface"); iface->get_child_by_index = - gst_videoaggregator_child_proxy_get_child_by_index; + gst_video_aggregator_child_proxy_get_child_by_index; iface->get_children_count = - gst_videoaggregator_child_proxy_get_children_count; + gst_video_aggregator_child_proxy_get_children_count; } /************************************** @@ -434,13 +435,13 @@ struct _GstVideoAggregatorPrivate * videoaggregator class in the _init to be able to set * the sink pad non-alpha caps. Using the G_DEFINE_TYPE there * seems to be no way of getting the real class being initialized */ -static void gst_videoaggregator_init (GstVideoAggregator * self, +static void gst_video_aggregator_init (GstVideoAggregator * self, GstVideoAggregatorClass * klass); -static void gst_videoaggregator_class_init (GstVideoAggregatorClass * klass); -static gpointer gst_videoaggregator_parent_class = NULL; +static void gst_video_aggregator_class_init (GstVideoAggregatorClass * klass); +static gpointer gst_video_aggregator_parent_class = NULL; GType -gst_videoaggregator_get_type (void) +gst_video_aggregator_get_type (void) { static volatile gsize g_define_type_id_volatile = 0; @@ -448,13 +449,13 @@ gst_videoaggregator_get_type (void) GType g_define_type_id = g_type_register_static_simple (GST_TYPE_AGGREGATOR, g_intern_static_string ("GstVideoAggregator"), sizeof (GstVideoAggregatorClass), - (GClassInitFunc) gst_videoaggregator_class_init, + (GClassInitFunc) gst_video_aggregator_class_init, sizeof (GstVideoAggregator), - (GInstanceInitFunc) gst_videoaggregator_init, + (GInstanceInitFunc) gst_video_aggregator_init, (GTypeFlags) G_TYPE_FLAG_ABSTRACT); { G_IMPLEMENT_INTERFACE (GST_TYPE_CHILD_PROXY, - gst_videoaggregator_child_proxy_init); + gst_video_aggregator_child_proxy_init); } g_once_init_leave (&g_define_type_id_volatile, g_define_type_id); } @@ -462,7 +463,7 @@ gst_videoaggregator_get_type (void) } static void -gst_videoaggreagator_find_best_format (GstVideoAggregator * vagg, +gst_video_aggregator_find_best_format (GstVideoAggregator * vagg, GstCaps * downstream_caps, GstVideoInfo * best_info, gboolean * at_least_one_alpha) { @@ -534,7 +535,7 @@ gst_videoaggreagator_find_best_format (GstVideoAggregator * vagg, /* WITH GST_VIDEO_AGGREGATOR_LOCK TAKEN */ static gboolean -gst_videoaggregator_src_setcaps (GstVideoAggregator * vagg, GstCaps * caps) +gst_video_aggregator_src_setcaps (GstVideoAggregator * vagg, GstCaps * caps) { GstAggregator *agg = GST_AGGREGATOR (vagg); gboolean ret = FALSE; @@ -558,7 +559,7 @@ gst_videoaggregator_src_setcaps (GstVideoAggregator * vagg, GstCaps * caps) GST_DEBUG_OBJECT (vagg, "Resetting frame counter because of framerate change"); } - gst_videoaggregator_reset_qos (vagg); + gst_video_aggregator_reset_qos (vagg); } vagg->info = info; @@ -583,7 +584,7 @@ done: } static GstCaps * -gst_videoaggregator_default_fixate_caps (GstVideoAggregator * vagg, +gst_video_aggregator_default_fixate_caps (GstVideoAggregator * vagg, GstCaps * caps) { gint best_width = -1, best_height = -1; @@ -644,7 +645,7 @@ gst_videoaggregator_default_fixate_caps (GstVideoAggregator * vagg, } static GstCaps * -gst_videoaggregator_default_update_caps (GstVideoAggregator * vagg, +gst_video_aggregator_default_update_caps (GstVideoAggregator * vagg, GstCaps * caps, GstCaps * filter) { GstVideoAggregatorClass *vagg_klass = GST_VIDEO_AGGREGATOR_GET_CLASS (vagg); @@ -691,7 +692,7 @@ gst_videoaggregator_default_update_caps (GstVideoAggregator * vagg, /* WITH GST_VIDEO_AGGREGATOR_LOCK TAKEN */ static gboolean -gst_videoaggregator_update_src_caps (GstVideoAggregator * vagg) +gst_video_aggregator_update_src_caps (GstVideoAggregator * vagg) { GstVideoAggregatorClass *vagg_klass = GST_VIDEO_AGGREGATOR_GET_CLASS (vagg); GstVideoAggregatorPadClass *vaggpad_klass = g_type_class_peek @@ -801,7 +802,7 @@ gst_videoaggregator_update_src_caps (GstVideoAggregator * vagg) } } - if (gst_videoaggregator_src_setcaps (vagg, caps)) { + if (gst_video_aggregator_src_setcaps (vagg, caps)) { if (vagg_klass->negotiated_caps) ret = GST_VIDEO_AGGREGATOR_GET_CLASS (vagg)->negotiated_caps (vagg, caps); @@ -821,7 +822,7 @@ done: } static gboolean -gst_videoaggregator_get_sinkpads_interlace_mode (GstVideoAggregator * vagg, +gst_video_aggregator_get_sinkpads_interlace_mode (GstVideoAggregator * vagg, GstVideoAggregatorPad * skip_pad, GstVideoInterlaceMode * mode) { GList *walk; @@ -841,7 +842,7 @@ gst_videoaggregator_get_sinkpads_interlace_mode (GstVideoAggregator * vagg, } static gboolean -gst_videoaggregator_pad_sink_setcaps (GstPad * pad, GstObject * parent, +gst_video_aggregator_pad_sink_setcaps (GstPad * pad, GstObject * parent, GstCaps * caps) { GstVideoAggregator *vagg; @@ -870,7 +871,7 @@ gst_videoaggregator_pad_sink_setcaps (GstPad * pad, GstObject * parent, has_mode = TRUE; } else { has_mode = - gst_videoaggregator_get_sinkpads_interlace_mode (vagg, vaggpad, + gst_video_aggregator_get_sinkpads_interlace_mode (vagg, vaggpad, &pads_mode); } @@ -896,7 +897,7 @@ beach: } static gboolean -gst_videoaggregator_caps_has_alpha (GstCaps * caps) +gst_video_aggregator_caps_has_alpha (GstCaps * caps) { guint size = gst_caps_get_size (caps); guint i; @@ -942,7 +943,7 @@ gst_videoaggregator_caps_has_alpha (GstCaps * caps) } static GstCaps * -gst_videoaggregator_pad_sink_getcaps (GstPad * pad, GstVideoAggregator * vagg, +gst_video_aggregator_pad_sink_getcaps (GstPad * pad, GstVideoAggregator * vagg, GstCaps * filter) { GstCaps *srccaps; @@ -962,10 +963,10 @@ gst_videoaggregator_pad_sink_getcaps (GstPad * pad, GstVideoAggregator * vagg, srccaps = gst_pad_peer_query_caps (srcpad, template_caps); srccaps = gst_caps_make_writable (srccaps); - has_alpha = gst_videoaggregator_caps_has_alpha (srccaps); + has_alpha = gst_video_aggregator_caps_has_alpha (srccaps); has_interlace_mode = - gst_videoaggregator_get_sinkpads_interlace_mode (vagg, NULL, + gst_video_aggregator_get_sinkpads_interlace_mode (vagg, NULL, &interlace_mode); n = gst_caps_get_size (srccaps); @@ -1011,7 +1012,7 @@ gst_videoaggregator_pad_sink_getcaps (GstPad * pad, GstVideoAggregator * vagg, } static void -gst_videoaggregator_update_qos (GstVideoAggregator * vagg, gdouble proportion, +gst_video_aggregator_update_qos (GstVideoAggregator * vagg, gdouble proportion, GstClockTimeDiff diff, GstClockTime timestamp) { gboolean live; @@ -1043,14 +1044,14 @@ gst_videoaggregator_update_qos (GstVideoAggregator * vagg, gdouble proportion, } static void -gst_videoaggregator_reset_qos (GstVideoAggregator * vagg) +gst_video_aggregator_reset_qos (GstVideoAggregator * vagg) { - gst_videoaggregator_update_qos (vagg, 0.5, 0, GST_CLOCK_TIME_NONE); + gst_video_aggregator_update_qos (vagg, 0.5, 0, GST_CLOCK_TIME_NONE); vagg->priv->qos_processed = vagg->priv->qos_dropped = 0; } static void -gst_videoaggregator_read_qos (GstVideoAggregator * vagg, gdouble * proportion, +gst_video_aggregator_read_qos (GstVideoAggregator * vagg, gdouble * proportion, GstClockTime * time) { GST_OBJECT_LOCK (vagg); @@ -1060,7 +1061,7 @@ gst_videoaggregator_read_qos (GstVideoAggregator * vagg, gdouble * proportion, } static void -gst_videoaggregator_reset (GstVideoAggregator * vagg) +gst_video_aggregator_reset (GstVideoAggregator * vagg) { GstAggregator *agg = GST_AGGREGATOR (vagg); GList *l; @@ -1072,7 +1073,7 @@ gst_videoaggregator_reset (GstVideoAggregator * vagg) agg->segment.position = -1; - gst_videoaggregator_reset_qos (vagg); + gst_video_aggregator_reset_qos (vagg); GST_OBJECT_LOCK (vagg); for (l = GST_ELEMENT (vagg)->sinkpads; l; l = l->next) { @@ -1089,7 +1090,7 @@ gst_videoaggregator_reset (GstVideoAggregator * vagg) #define GST_FLOW_NEEDS_DATA GST_FLOW_CUSTOM_ERROR static gint -gst_videoaggregator_fill_queues (GstVideoAggregator * vagg, +gst_video_aggregator_fill_queues (GstVideoAggregator * vagg, GstClockTime output_start_running_time, GstClockTime output_end_running_time) { @@ -1331,7 +1332,7 @@ clean_pad (GstVideoAggregator * vagg, GstVideoAggregatorPad * pad) } static GstFlowReturn -gst_videoaggregator_do_aggregate (GstVideoAggregator * vagg, +gst_video_aggregator_do_aggregate (GstVideoAggregator * vagg, GstClockTime output_start_time, GstClockTime output_end_time, GstBuffer ** outbuf) { @@ -1378,7 +1379,7 @@ gst_videoaggregator_do_aggregate (GstVideoAggregator * vagg, /* Perform qos calculations before processing the next frame. Returns TRUE if * the frame should be processed, FALSE if the frame can be dropped entirely */ static gint64 -gst_videoaggregator_do_qos (GstVideoAggregator * vagg, GstClockTime timestamp) +gst_video_aggregator_do_qos (GstVideoAggregator * vagg, GstClockTime timestamp) { GstAggregator *agg = GST_AGGREGATOR (vagg); GstClockTime qostime, earliest_time; @@ -1392,7 +1393,7 @@ gst_videoaggregator_do_qos (GstVideoAggregator * vagg, GstClockTime timestamp) } /* get latest QoS observation values */ - gst_videoaggregator_read_qos (vagg, &proportion, &earliest_time); + gst_video_aggregator_read_qos (vagg, &proportion, &earliest_time); /* skip qos if we have no observation (yet) => process frame */ if (G_UNLIKELY (!GST_CLOCK_TIME_IS_VALID (earliest_time))) { @@ -1419,7 +1420,7 @@ gst_videoaggregator_do_qos (GstVideoAggregator * vagg, GstClockTime timestamp) } static GstClockTime -gst_videoaggregator_get_next_time (GstAggregator * agg) +gst_video_aggregator_get_next_time (GstAggregator * agg) { GstClockTime next_time; @@ -1440,7 +1441,7 @@ gst_videoaggregator_get_next_time (GstAggregator * agg) } static GstFlowReturn -gst_videoaggregator_check_reconfigure (GstVideoAggregator * vagg, +gst_video_aggregator_check_reconfigure (GstVideoAggregator * vagg, gboolean timeout) { GstAggregator *agg = (GstAggregator *) vagg; @@ -1449,7 +1450,7 @@ gst_videoaggregator_check_reconfigure (GstVideoAggregator * vagg, || gst_pad_check_reconfigure (GST_AGGREGATOR_SRC_PAD (vagg))) { gboolean ret; - ret = gst_videoaggregator_update_src_caps (vagg); + ret = gst_video_aggregator_update_src_caps (vagg); if (!ret) { if (timeout && gst_pad_needs_reconfigure (GST_AGGREGATOR_SRC_PAD (vagg))) { guint64 frame_duration; @@ -1490,7 +1491,7 @@ gst_videoaggregator_check_reconfigure (GstVideoAggregator * vagg, } static GstFlowReturn -gst_videoaggregator_aggregate (GstAggregator * agg, gboolean timeout) +gst_video_aggregator_aggregate (GstAggregator * agg, gboolean timeout) { GstVideoAggregator *vagg = GST_VIDEO_AGGREGATOR (agg); GstClockTime output_start_time, output_end_time; @@ -1501,7 +1502,7 @@ gst_videoaggregator_aggregate (GstAggregator * agg, gboolean timeout) GST_VIDEO_AGGREGATOR_LOCK (vagg); - flow_ret = gst_videoaggregator_check_reconfigure (vagg, timeout); + flow_ret = gst_video_aggregator_check_reconfigure (vagg, timeout); if (flow_ret != GST_FLOW_OK) { if (flow_ret == GST_FLOW_NEEDS_DATA) flow_ret = GST_FLOW_OK; @@ -1542,7 +1543,7 @@ gst_videoaggregator_aggregate (GstAggregator * agg, gboolean timeout) flow_ret = GST_FLOW_EOS; } else { flow_ret = - gst_videoaggregator_fill_queues (vagg, output_start_running_time, + gst_video_aggregator_fill_queues (vagg, output_start_running_time, output_end_running_time); } @@ -1566,9 +1567,9 @@ gst_videoaggregator_aggregate (GstAggregator * agg, gboolean timeout) GST_TIME_ARGS (output_start_running_time), GST_TIME_ARGS (output_end_running_time)); - jitter = gst_videoaggregator_do_qos (vagg, output_start_time); + jitter = gst_video_aggregator_do_qos (vagg, output_start_time); if (jitter <= 0) { - flow_ret = gst_videoaggregator_do_aggregate (vagg, output_start_time, + flow_ret = gst_video_aggregator_do_aggregate (vagg, output_start_time, output_end_time, &outbuf); if (flow_ret != GST_FLOW_OK) goto done; @@ -1633,7 +1634,8 @@ unlock_and_return: * cases work at least somewhat. */ static gboolean -gst_videoaggregator_query_duration (GstVideoAggregator * vagg, GstQuery * query) +gst_video_aggregator_query_duration (GstVideoAggregator * vagg, + GstQuery * query) { GValue item = { 0 }; gint64 max; @@ -1704,7 +1706,7 @@ gst_videoaggregator_query_duration (GstVideoAggregator * vagg, GstQuery * query) } static gboolean -gst_videoaggregator_src_query (GstAggregator * agg, GstQuery * query) +gst_video_aggregator_src_query (GstAggregator * agg, GstQuery * query) { GstVideoAggregator *vagg = GST_VIDEO_AGGREGATOR (agg); gboolean res = FALSE; @@ -1729,11 +1731,11 @@ gst_videoaggregator_src_query (GstAggregator * agg, GstQuery * query) break; } case GST_QUERY_DURATION: - res = gst_videoaggregator_query_duration (vagg, query); + res = gst_video_aggregator_query_duration (vagg, query); break; case GST_QUERY_LATENCY: res = - GST_AGGREGATOR_CLASS (gst_videoaggregator_parent_class)->src_query + GST_AGGREGATOR_CLASS (gst_video_aggregator_parent_class)->src_query (agg, query); if (res) { @@ -1742,7 +1744,7 @@ gst_videoaggregator_src_query (GstAggregator * agg, GstQuery * query) break; default: res = - GST_AGGREGATOR_CLASS (gst_videoaggregator_parent_class)->src_query + GST_AGGREGATOR_CLASS (gst_video_aggregator_parent_class)->src_query (agg, query); break; } @@ -1750,7 +1752,7 @@ gst_videoaggregator_src_query (GstAggregator * agg, GstQuery * query) } static gboolean -gst_videoaggregator_src_event (GstAggregator * agg, GstEvent * event) +gst_video_aggregator_src_event (GstAggregator * agg, GstEvent * event) { GstVideoAggregator *vagg = GST_VIDEO_AGGREGATOR (agg); @@ -1763,7 +1765,7 @@ gst_videoaggregator_src_event (GstAggregator * agg, GstEvent * event) gdouble proportion; gst_event_parse_qos (event, &type, &proportion, &diff, ×tamp); - gst_videoaggregator_update_qos (vagg, proportion, diff, timestamp); + gst_video_aggregator_update_qos (vagg, proportion, diff, timestamp); break; } case GST_EVENT_SEEK: @@ -1775,12 +1777,12 @@ gst_videoaggregator_src_event (GstAggregator * agg, GstEvent * event) } return - GST_AGGREGATOR_CLASS (gst_videoaggregator_parent_class)->src_event (agg, + GST_AGGREGATOR_CLASS (gst_video_aggregator_parent_class)->src_event (agg, event); } static GstFlowReturn -gst_videoaggregator_flush (GstAggregator * agg) +gst_video_aggregator_flush (GstAggregator * agg) { GList *l; gdouble abs_rate; @@ -1810,12 +1812,12 @@ gst_videoaggregator_flush (GstAggregator * agg) vagg->priv->ts_offset = 0; vagg->priv->nframes = 0; - gst_videoaggregator_reset_qos (vagg); + gst_video_aggregator_reset_qos (vagg); return GST_FLOW_OK; } static gboolean -gst_videoaggregator_sink_event (GstAggregator * agg, GstAggregatorPad * bpad, +gst_video_aggregator_sink_event (GstAggregator * agg, GstAggregatorPad * bpad, GstEvent * event) { GstVideoAggregator *vagg = GST_VIDEO_AGGREGATOR (agg); @@ -1832,7 +1834,7 @@ gst_videoaggregator_sink_event (GstAggregator * agg, GstAggregatorPad * bpad, gst_event_parse_caps (event, &caps); ret = - gst_videoaggregator_pad_sink_setcaps (GST_PAD (pad), + gst_video_aggregator_pad_sink_setcaps (GST_PAD (pad), GST_OBJECT (vagg), caps); gst_event_unref (event); event = NULL; @@ -1843,7 +1845,7 @@ gst_videoaggregator_sink_event (GstAggregator * agg, GstAggregatorPad * bpad, gst_event_copy_segment (event, &seg); g_assert (seg.format == GST_FORMAT_TIME); - gst_videoaggregator_reset_qos (vagg); + gst_video_aggregator_reset_qos (vagg); break; } default: @@ -1851,14 +1853,14 @@ gst_videoaggregator_sink_event (GstAggregator * agg, GstAggregatorPad * bpad, } if (event != NULL) - return GST_AGGREGATOR_CLASS (gst_videoaggregator_parent_class)->sink_event + return GST_AGGREGATOR_CLASS (gst_video_aggregator_parent_class)->sink_event (agg, bpad, event); return ret; } static gboolean -gst_videoaggregator_start (GstAggregator * agg) +gst_video_aggregator_start (GstAggregator * agg) { GstVideoAggregator *vagg = GST_VIDEO_AGGREGATOR (agg); @@ -1868,18 +1870,18 @@ gst_videoaggregator_start (GstAggregator * agg) } static gboolean -gst_videoaggregator_stop (GstAggregator * agg) +gst_video_aggregator_stop (GstAggregator * agg) { GstVideoAggregator *vagg = GST_VIDEO_AGGREGATOR (agg); - gst_videoaggregator_reset (vagg); + gst_video_aggregator_reset (vagg); return TRUE; } /* GstElement vmethods */ static GstPad * -gst_videoaggregator_request_new_pad (GstElement * element, +gst_video_aggregator_request_new_pad (GstElement * element, GstPadTemplate * templ, const gchar * req_name, const GstCaps * caps) { GstVideoAggregator *vagg; @@ -1888,7 +1890,7 @@ gst_videoaggregator_request_new_pad (GstElement * element, vagg = GST_VIDEO_AGGREGATOR (element); vaggpad = (GstVideoAggregatorPad *) - GST_ELEMENT_CLASS (gst_videoaggregator_parent_class)->request_new_pad + GST_ELEMENT_CLASS (gst_video_aggregator_parent_class)->request_new_pad (element, templ, req_name, caps); if (vaggpad == NULL) @@ -1909,7 +1911,7 @@ gst_videoaggregator_request_new_pad (GstElement * element, } static void -gst_videoaggregator_release_pad (GstElement * element, GstPad * pad) +gst_video_aggregator_release_pad (GstElement * element, GstPad * pad) { GstVideoAggregator *vagg = NULL; GstVideoAggregatorPad *vaggpad; @@ -1925,15 +1927,15 @@ gst_videoaggregator_release_pad (GstElement * element, GstPad * pad) GST_OBJECT_UNLOCK (vagg); if (last_pad) - gst_videoaggregator_reset (vagg); + gst_video_aggregator_reset (vagg); gst_buffer_replace (&vaggpad->buffer, NULL); gst_child_proxy_child_removed (GST_CHILD_PROXY (vagg), G_OBJECT (vaggpad), GST_OBJECT_NAME (vaggpad)); - GST_ELEMENT_CLASS (gst_videoaggregator_parent_class)->release_pad (GST_ELEMENT - (vagg), pad); + GST_ELEMENT_CLASS (gst_video_aggregator_parent_class)->release_pad + (GST_ELEMENT (vagg), pad); gst_pad_mark_reconfigure (GST_AGGREGATOR_SRC_PAD (vagg)); @@ -1942,7 +1944,7 @@ gst_videoaggregator_release_pad (GstElement * element, GstPad * pad) } static GstFlowReturn -gst_videoaggregator_get_output_buffer (GstVideoAggregator * videoaggregator, +gst_video_aggregator_get_output_buffer (GstVideoAggregator * videoaggregator, GstBuffer ** outbuf) { guint outsize; @@ -1960,7 +1962,7 @@ gst_videoaggregator_get_output_buffer (GstVideoAggregator * videoaggregator, } static gboolean -gst_videoaggregator_pad_sink_acceptcaps (GstPad * pad, +gst_video_aggregator_pad_sink_acceptcaps (GstPad * pad, GstVideoAggregator * vagg, GstCaps * caps) { gboolean ret; @@ -2011,7 +2013,7 @@ gst_videoaggregator_pad_sink_acceptcaps (GstPad * pad, } static gboolean -gst_videoaggregator_sink_query (GstAggregator * agg, GstAggregatorPad * bpad, +gst_video_aggregator_sink_query (GstAggregator * agg, GstAggregatorPad * bpad, GstQuery * query) { GstVideoAggregator *vagg = GST_VIDEO_AGGREGATOR (agg); @@ -2024,7 +2026,8 @@ gst_videoaggregator_sink_query (GstAggregator * agg, GstAggregatorPad * bpad, GstCaps *filter, *caps; gst_query_parse_caps (query, &filter); - caps = gst_videoaggregator_pad_sink_getcaps (GST_PAD (pad), vagg, filter); + caps = + gst_video_aggregator_pad_sink_getcaps (GST_PAD (pad), vagg, filter); gst_query_set_caps_result (query, caps); gst_caps_unref (caps); ret = TRUE; @@ -2035,14 +2038,15 @@ gst_videoaggregator_sink_query (GstAggregator * agg, GstAggregatorPad * bpad, GstCaps *caps; gst_query_parse_accept_caps (query, &caps); - ret = gst_videoaggregator_pad_sink_acceptcaps (GST_PAD (pad), vagg, caps); + ret = + gst_video_aggregator_pad_sink_acceptcaps (GST_PAD (pad), vagg, caps); gst_query_set_accept_caps_result (query, ret); ret = TRUE; break; } default: ret = - GST_AGGREGATOR_CLASS (gst_videoaggregator_parent_class)->sink_query + GST_AGGREGATOR_CLASS (gst_video_aggregator_parent_class)->sink_query (agg, bpad, query); break; } @@ -2051,27 +2055,27 @@ gst_videoaggregator_sink_query (GstAggregator * agg, GstAggregatorPad * bpad, /* GObject vmethods */ static void -gst_videoaggregator_finalize (GObject * o) +gst_video_aggregator_finalize (GObject * o) { GstVideoAggregator *vagg = GST_VIDEO_AGGREGATOR (o); g_mutex_clear (&vagg->priv->lock); - G_OBJECT_CLASS (gst_videoaggregator_parent_class)->finalize (o); + G_OBJECT_CLASS (gst_video_aggregator_parent_class)->finalize (o); } static void -gst_videoaggregator_dispose (GObject * o) +gst_video_aggregator_dispose (GObject * o) { GstVideoAggregator *vagg = GST_VIDEO_AGGREGATOR (o); gst_caps_replace (&vagg->priv->current_caps, NULL); - G_OBJECT_CLASS (gst_videoaggregator_parent_class)->dispose (o); + G_OBJECT_CLASS (gst_video_aggregator_parent_class)->dispose (o); } static void -gst_videoaggregator_get_property (GObject * object, +gst_video_aggregator_get_property (GObject * object, guint prop_id, GValue * value, GParamSpec * pspec) { switch (prop_id) { @@ -2082,7 +2086,7 @@ gst_videoaggregator_get_property (GObject * object, } static void -gst_videoaggregator_set_property (GObject * object, +gst_video_aggregator_set_property (GObject * object, guint prop_id, const GValue * value, GParamSpec * pspec) { switch (prop_id) { @@ -2094,45 +2098,45 @@ gst_videoaggregator_set_property (GObject * object, /* GObject boilerplate */ static void -gst_videoaggregator_class_init (GstVideoAggregatorClass * klass) +gst_video_aggregator_class_init (GstVideoAggregatorClass * klass) { GObjectClass *gobject_class = (GObjectClass *) klass; GstElementClass *gstelement_class = (GstElementClass *) klass; GstAggregatorClass *agg_class = (GstAggregatorClass *) klass; - GST_DEBUG_CATEGORY_INIT (gst_videoaggregator_debug, "videoaggregator", 0, + GST_DEBUG_CATEGORY_INIT (gst_video_aggregator_debug, "videoaggregator", 0, "base video aggregator"); - gst_videoaggregator_parent_class = g_type_class_peek_parent (klass); + gst_video_aggregator_parent_class = g_type_class_peek_parent (klass); g_type_class_add_private (klass, sizeof (GstVideoAggregatorPrivate)); - gobject_class->finalize = gst_videoaggregator_finalize; - gobject_class->dispose = gst_videoaggregator_dispose; + gobject_class->finalize = gst_video_aggregator_finalize; + gobject_class->dispose = gst_video_aggregator_dispose; - gobject_class->get_property = gst_videoaggregator_get_property; - gobject_class->set_property = gst_videoaggregator_set_property; + gobject_class->get_property = gst_video_aggregator_get_property; + gobject_class->set_property = gst_video_aggregator_set_property; gstelement_class->request_new_pad = - GST_DEBUG_FUNCPTR (gst_videoaggregator_request_new_pad); + GST_DEBUG_FUNCPTR (gst_video_aggregator_request_new_pad); gstelement_class->release_pad = - GST_DEBUG_FUNCPTR (gst_videoaggregator_release_pad); + GST_DEBUG_FUNCPTR (gst_video_aggregator_release_pad); agg_class->sinkpads_type = GST_TYPE_VIDEO_AGGREGATOR_PAD; - agg_class->start = gst_videoaggregator_start; - agg_class->stop = gst_videoaggregator_stop; - agg_class->sink_query = gst_videoaggregator_sink_query; - agg_class->sink_event = gst_videoaggregator_sink_event; - agg_class->flush = gst_videoaggregator_flush; - agg_class->aggregate = gst_videoaggregator_aggregate; - agg_class->src_event = gst_videoaggregator_src_event; - agg_class->src_query = gst_videoaggregator_src_query; - agg_class->get_next_time = gst_videoaggregator_get_next_time; + agg_class->start = gst_video_aggregator_start; + agg_class->stop = gst_video_aggregator_stop; + agg_class->sink_query = gst_video_aggregator_sink_query; + agg_class->sink_event = gst_video_aggregator_sink_event; + agg_class->flush = gst_video_aggregator_flush; + agg_class->aggregate = gst_video_aggregator_aggregate; + agg_class->src_event = gst_video_aggregator_src_event; + agg_class->src_query = gst_video_aggregator_src_query; + agg_class->get_next_time = gst_video_aggregator_get_next_time; - klass->find_best_format = gst_videoaggreagator_find_best_format; - klass->get_output_buffer = gst_videoaggregator_get_output_buffer; - klass->update_caps = gst_videoaggregator_default_update_caps; - klass->fixate_caps = gst_videoaggregator_default_fixate_caps; + klass->find_best_format = gst_video_aggregator_find_best_format; + klass->get_output_buffer = gst_video_aggregator_get_output_buffer; + klass->update_caps = gst_video_aggregator_default_update_caps; + klass->fixate_caps = gst_video_aggregator_default_fixate_caps; /* Register the pad class */ g_type_class_ref (GST_TYPE_VIDEO_AGGREGATOR_PAD); @@ -2211,7 +2215,7 @@ _get_non_alpha_caps_from_template (GstVideoAggregatorClass * klass) static GMutex sink_caps_mutex; static void -gst_videoaggregator_init (GstVideoAggregator * vagg, +gst_video_aggregator_init (GstVideoAggregator * vagg, GstVideoAggregatorClass * klass) { @@ -2230,5 +2234,5 @@ gst_videoaggregator_init (GstVideoAggregator * vagg, } g_mutex_unlock (&sink_caps_mutex); - gst_videoaggregator_reset (vagg); + gst_video_aggregator_reset (vagg); } diff --git a/gst-libs/gst/video/gstvideoaggregator.h b/gst-libs/gst/video/gstvideoaggregator.h index 86df0b6294..6204bdd037 100644 --- a/gst-libs/gst/video/gstvideoaggregator.h +++ b/gst-libs/gst/video/gstvideoaggregator.h @@ -32,7 +32,7 @@ G_BEGIN_DECLS -#define GST_TYPE_VIDEO_AGGREGATOR (gst_videoaggregator_get_type()) +#define GST_TYPE_VIDEO_AGGREGATOR (gst_video_aggregator_get_type()) #define GST_VIDEO_AGGREGATOR(obj) \ (G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_VIDEO_AGGREGATOR, GstVideoAggregator)) #define GST_VIDEO_AGGREGATOR_CLASS(klass) \ @@ -118,7 +118,7 @@ struct _GstVideoAggregatorClass gpointer _gst_reserved[GST_PADDING_LARGE]; }; -GType gst_videoaggregator_get_type (void); +GType gst_video_aggregator_get_type (void); G_END_DECLS #endif /* __GST_VIDEO_AGGREGATOR_H__ */ From 0934af6cce33d3759dd4fc13ff315073291446fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Olivier=20Cr=C3=AAte?= Date: Sat, 14 May 2016 11:56:59 +0200 Subject: [PATCH 261/381] videoaggregator: Don't wait if input buffer is after output If the input buffer is after the end of the output buffer, then waiting for more data won't help. We will never get an input buffer for this point. This fixes compositing of streams from rtspsrc. https://bugzilla.gnome.org/show_bug.cgi?id=766422 --- gst-libs/gst/video/gstvideoaggregator.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index 49ab5ae84d..28b2283557 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -1147,8 +1147,7 @@ gst_video_aggregator_fill_queues (GstVideoAggregator * vagg, "output_end_running_time. Keeping previous buffer"); } else { GST_DEBUG_OBJECT (pad, "buffer duration is -1, start_time >= " - "output_end_running_time. No previous buffer, need more data"); - need_more_data = TRUE; + "output_end_running_time. No previous buffer."); } gst_buffer_unref (buf); continue; From 396932f412f01de86351d4f6deb1f0c9b2f84c68 Mon Sep 17 00:00:00 2001 From: Guillaume Desmottes Date: Thu, 19 May 2016 16:55:31 +0300 Subject: [PATCH 262/381] glmosaic: fix shader leak gst_gl_mosaic_init_shader() is called twice with test_glmosaic so the first shader was leaked. https://bugzilla.gnome.org/show_bug.cgi?id=766661 --- ext/gl/gstglmosaic.c | 1 + 1 file changed, 1 insertion(+) diff --git a/ext/gl/gstglmosaic.c b/ext/gl/gstglmosaic.c index 377f71869f..bcf38331be 100644 --- a/ext/gl/gstglmosaic.c +++ b/ext/gl/gstglmosaic.c @@ -187,6 +187,7 @@ gst_gl_mosaic_init_shader (GstGLMixer * mixer, GstCaps * outcaps) { GstGLMosaic *mosaic = GST_GL_MOSAIC (mixer); + g_clear_object (&mosaic->shader); //blocking call, wait the opengl thread has compiled the shader return gst_gl_context_gen_shader (GST_GL_BASE_MIXER (mixer)->context, mosaic_v_src, mosaic_f_src, &mosaic->shader); From 11bb4454a8ba60637fb224cdb944c58a0610f5d2 Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Tue, 24 May 2016 23:39:27 +1000 Subject: [PATCH 263/381] glvideomixer: fix race retrieving the GL context from the display _get_gl_context() can be called concurrently from either propose_allocation() or decide_allocation(). If it so happens that this happens at the same time, the check for whether we already had a GL context was outside the lock. Inside the lock and loop, the first thing that happens is that we unref the current GL context (if valid) as if there was a conflict adding it to the display. If the timing was unlucky, subsequent use of the GL context would be referencing an already unreffed GL context object resulting in a critical: g_object_ref: assertion 'object->ref_count > 0' failed https://bugzilla.gnome.org/show_bug.cgi?id=766703 --- ext/gl/gstglbasemixer.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ext/gl/gstglbasemixer.c b/ext/gl/gstglbasemixer.c index 9d2851e33d..410a42022a 100644 --- a/ext/gl/gstglbasemixer.c +++ b/ext/gl/gstglbasemixer.c @@ -195,8 +195,8 @@ _get_gl_context (GstGLBaseMixer * mix) _find_local_gl_context (mix); + GST_OBJECT_LOCK (mix->display); if (!mix->context) { - GST_OBJECT_LOCK (mix->display); do { if (mix->context) { gst_object_unref (mix->context); @@ -213,8 +213,8 @@ _get_gl_context (GstGLBaseMixer * mix) } } } while (!gst_gl_display_add_context (mix->display, mix->context)); - GST_OBJECT_UNLOCK (mix->display); } + GST_OBJECT_UNLOCK (mix->display); { GstGLAPI current_gl_api = gst_gl_context_get_gl_api (mix->context); From f2c0e89021d67562476a61d89185f9071ac24ffa Mon Sep 17 00:00:00 2001 From: Guillaume Desmottes Date: Tue, 17 May 2016 17:14:49 +0300 Subject: [PATCH 264/381] gst-libs: gl, video: use MAY_BE_LEAKED flag https://bugzilla.gnome.org/show_bug.cgi?id=767162 --- gst-libs/gst/video/gstvideoaggregator.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index 28b2283557..9046252958 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -2230,6 +2230,10 @@ gst_video_aggregator_init (GstVideoAggregator * vagg, g_mutex_lock (&sink_caps_mutex); if (klass->sink_non_alpha_caps == NULL) { klass->sink_non_alpha_caps = _get_non_alpha_caps_from_template (klass); + + /* The caps is cached */ + GST_MINI_OBJECT_FLAG_SET (klass->sink_non_alpha_caps, + GST_MINI_OBJECT_FLAG_MAY_BE_LEAKED); } g_mutex_unlock (&sink_caps_mutex); From 8b0c9e991818ee3a0e844fae62b48c32db776a0b Mon Sep 17 00:00:00 2001 From: Guillaume Desmottes Date: Thu, 19 May 2016 11:19:20 +0300 Subject: [PATCH 265/381] fix event leaks in tests The events are supposed to be unreffed when finishing the test, not reffed. https://bugzilla.gnome.org/show_bug.cgi?id=766663 --- tests/check/elements/compositor.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/check/elements/compositor.c b/tests/check/elements/compositor.c index 258d1d96cd..e9f016543f 100644 --- a/tests/check/elements/compositor.c +++ b/tests/check/elements/compositor.c @@ -754,7 +754,7 @@ GST_START_TEST (test_play_twice) /* cleanup */ g_main_loop_unref (main_loop); gst_consistency_checker_free (consist); - gst_event_ref (play_seek_event); + gst_event_unref (play_seek_event); gst_bus_remove_signal_watch (bus); gst_object_unref (bus); gst_object_unref (bin); @@ -857,7 +857,7 @@ GST_START_TEST (test_play_twice_then_add_and_play_again) /* cleanup */ g_main_loop_unref (main_loop); - gst_event_ref (play_seek_event); + gst_event_unref (play_seek_event); gst_consistency_checker_free (consist); gst_bus_remove_signal_watch (bus); gst_object_unref (bus); From 9b7d248982bfb134f5c03e0d053054c8660d8faf Mon Sep 17 00:00:00 2001 From: Nirbheek Chauhan Date: Mon, 11 Jul 2016 19:21:11 +0530 Subject: [PATCH 266/381] Fix various gboolean vs GstFlowReturn problems Caught by building with MSVC which gave warnings for these --- gst-libs/gst/video/gstvideoaggregator.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index 9046252958..1d40854d15 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -133,7 +133,7 @@ gst_video_aggregator_pad_set_property (GObject * object, guint prop_id, gst_object_unref (vagg); } -static gboolean +static GstFlowReturn _flush_pad (GstAggregatorPad * aggpad, GstAggregator * aggregator) { GstVideoAggregator *vagg = GST_VIDEO_AGGREGATOR (aggregator); @@ -144,7 +144,7 @@ _flush_pad (GstAggregatorPad * aggpad, GstAggregator * aggregator) pad->priv->start_time = -1; pad->priv->end_time = -1; - return TRUE; + return GST_FLOW_OK; } static gboolean From c8c016ed177353721a43cc45885828953471908e Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Tue, 12 Jul 2016 00:30:22 +1000 Subject: [PATCH 267/381] gl: use GLMemory for accessing textures everywhere This simplifies and consolidates a lot of duplicated code creating and modifying textures. --- ext/gl/gstglmixer.c | 4 ++-- ext/gl/gstglmixer.h | 2 +- ext/gl/gstglmosaic.c | 7 ++++--- ext/gl/gstglmosaic.h | 1 + ext/gl/gstglvideomixer.c | 6 +++--- 5 files changed, 11 insertions(+), 9 deletions(-) diff --git a/ext/gl/gstglmixer.c b/ext/gl/gstglmixer.c index 764a53c33e..f9c22a1d2e 100644 --- a/ext/gl/gstglmixer.c +++ b/ext/gl/gstglmixer.c @@ -631,7 +631,7 @@ _upload_frames (GstAggregator * agg, GstAggregatorPad * agg_pad, gboolean gst_gl_mixer_process_textures (GstGLMixer * mix, GstBuffer * outbuf) { - guint out_tex; + GstGLMemory *out_tex; gboolean res = TRUE; GstVideoFrame out_frame; GstVideoAggregator *vagg = GST_VIDEO_AGGREGATOR (mix); @@ -645,7 +645,7 @@ gst_gl_mixer_process_textures (GstGLMixer * mix, GstBuffer * outbuf) return FALSE; } - out_tex = *(guint *) out_frame.data[0]; + out_tex = (GstGLMemory *) out_frame.map[0].memory; if (!gst_aggregator_iterate_sinkpads (GST_AGGREGATOR (mix), (GstAggregatorPadForeachFunc) _upload_frames, NULL)) diff --git a/ext/gl/gstglmixer.h b/ext/gl/gstglmixer.h index 81976a748d..de5a3dc4ef 100644 --- a/ext/gl/gstglmixer.h +++ b/ext/gl/gstglmixer.h @@ -78,7 +78,7 @@ typedef gboolean (*GstGLMixerSetCaps) (GstGLMixer* mixer, GstCaps* outcaps); typedef void (*GstGLMixerReset) (GstGLMixer *mixer); typedef gboolean (*GstGLMixerProcessFunc) (GstGLMixer *mix, GstBuffer *outbuf); -typedef gboolean (*GstGLMixerProcessTextures) (GstGLMixer *mix, guint out_tex); +typedef gboolean (*GstGLMixerProcessTextures) (GstGLMixer *mix, GstGLMemory *out_tex); struct _GstGLMixer { diff --git a/ext/gl/gstglmosaic.c b/ext/gl/gstglmosaic.c index bcf38331be..701435f804 100644 --- a/ext/gl/gstglmosaic.c +++ b/ext/gl/gstglmosaic.c @@ -70,7 +70,7 @@ static gboolean gst_gl_mosaic_init_shader (GstGLMixer * mixer, GstCaps * outcaps); static gboolean gst_gl_mosaic_process_textures (GstGLMixer * mixer, - guint out_tex); + GstGLMemory * out_tex); static void gst_gl_mosaic_callback (gpointer stuff); //vertex source @@ -194,7 +194,7 @@ gst_gl_mosaic_init_shader (GstGLMixer * mixer, GstCaps * outcaps) } static gboolean -gst_gl_mosaic_process_textures (GstGLMixer * mix, guint out_tex) +gst_gl_mosaic_process_textures (GstGLMixer * mix, GstGLMemory * out_tex) { GstGLMosaic *mosaic = GST_GL_MOSAIC (mix); @@ -202,7 +202,8 @@ gst_gl_mosaic_process_textures (GstGLMixer * mix, guint out_tex) gst_gl_context_use_fbo_v2 (GST_GL_BASE_MIXER (mix)->context, GST_VIDEO_INFO_WIDTH (&GST_VIDEO_AGGREGATOR (mix)->info), GST_VIDEO_INFO_HEIGHT (&GST_VIDEO_AGGREGATOR (mix)->info), mix->fbo, - mix->depthbuffer, out_tex, gst_gl_mosaic_callback, (gpointer) mosaic); + mix->depthbuffer, out_tex->tex_id, gst_gl_mosaic_callback, + (gpointer) mosaic); return TRUE; } diff --git a/ext/gl/gstglmosaic.h b/ext/gl/gstglmosaic.h index e0b340cc28..40f5d6c33c 100644 --- a/ext/gl/gstglmosaic.h +++ b/ext/gl/gstglmosaic.h @@ -40,6 +40,7 @@ struct _GstGLMosaic GstGLMixer mixer; GstGLShader *shader; + GstGLMemory *out_tex; }; struct _GstGLMosaicClass diff --git a/ext/gl/gstglvideomixer.c b/ext/gl/gstglvideomixer.c index 83905d0e68..176d27e547 100644 --- a/ext/gl/gstglvideomixer.c +++ b/ext/gl/gstglvideomixer.c @@ -474,7 +474,7 @@ static gboolean gst_gl_video_mixer_init_shader (GstGLMixer * mixer, GstCaps * outcaps); static gboolean gst_gl_video_mixer_process_textures (GstGLMixer * mixer, - guint out_tex); + GstGLMemory * out_tex); static void gst_gl_video_mixer_callback (gpointer stuff); /* *INDENT-OFF* */ @@ -1156,7 +1156,7 @@ gst_gl_video_mixer_init_shader (GstGLMixer * mixer, GstCaps * outcaps) } static gboolean -gst_gl_video_mixer_process_textures (GstGLMixer * mix, guint out_tex) +gst_gl_video_mixer_process_textures (GstGLMixer * mix, GstGLMemory * out_tex) { GstGLVideoMixer *video_mixer = GST_GL_VIDEO_MIXER (mix); @@ -1164,7 +1164,7 @@ gst_gl_video_mixer_process_textures (GstGLMixer * mix, guint out_tex) GST_VIDEO_INFO_WIDTH (&GST_VIDEO_AGGREGATOR (mix)->info), GST_VIDEO_INFO_HEIGHT (&GST_VIDEO_AGGREGATOR (mix)->info), mix->fbo, mix->depthbuffer, - out_tex, gst_gl_video_mixer_callback, (gpointer) video_mixer); + out_tex->tex_id, gst_gl_video_mixer_callback, (gpointer) video_mixer); return TRUE; } From 5a74878988db8d82eba1da78cb325018b031c080 Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Tue, 12 Jul 2016 12:59:57 +1000 Subject: [PATCH 268/381] glframebuffer: rewrite for a more consistent API Facilities are given to create fbo's and attach GL memory (renderbuffers or textures). It also keeps track of the renderable size for use with effective use with glViewport(). --- ext/gl/gstglmixer.c | 38 ++++++++++++++++++++------------------ ext/gl/gstglmixer.h | 3 +-- ext/gl/gstglmosaic.c | 26 ++++++++++++++++++-------- ext/gl/gstglvideomixer.c | 27 +++++++++++++++++++-------- ext/gl/gstglvideomixer.h | 1 + 5 files changed, 59 insertions(+), 36 deletions(-) diff --git a/ext/gl/gstglmixer.c b/ext/gl/gstglmixer.c index f9c22a1d2e..3fc5aa6f50 100644 --- a/ext/gl/gstglmixer.c +++ b/ext/gl/gstglmixer.c @@ -410,8 +410,6 @@ static void gst_gl_mixer_init (GstGLMixer * mix) { mix->priv = GST_GL_MIXER_GET_PRIVATE (mix); - mix->fbo = 0; - mix->depthbuffer = 0; mix->priv->gl_resource_ready = FALSE; g_mutex_init (&mix->priv->gl_resource_lock); @@ -512,33 +510,38 @@ gst_gl_mixer_get_output_buffer (GstVideoAggregator * videoaggregator, return ret; } +static void +_mixer_create_fbo (GstGLContext * context, GstGLMixer * mix) +{ + GstVideoAggregator *vagg = GST_VIDEO_AGGREGATOR (mix); + guint out_width = GST_VIDEO_INFO_WIDTH (&vagg->info); + guint out_height = GST_VIDEO_INFO_HEIGHT (&vagg->info); + + mix->fbo = + gst_gl_framebuffer_new_with_default_depth (context, out_width, + out_height); +} + static gboolean gst_gl_mixer_decide_allocation (GstGLBaseMixer * base_mix, GstQuery * query) { GstGLMixer *mix = GST_GL_MIXER (base_mix); GstGLMixerClass *mixer_class = GST_GL_MIXER_GET_CLASS (mix); - GstVideoAggregator *vagg = GST_VIDEO_AGGREGATOR (mix); GstGLContext *context = base_mix->context; GstBufferPool *pool = NULL; GstStructure *config; GstCaps *caps; guint min, max, size; gboolean update_pool; - guint out_width, out_height; - - out_width = GST_VIDEO_INFO_WIDTH (&vagg->info); - out_height = GST_VIDEO_INFO_HEIGHT (&vagg->info); g_mutex_lock (&mix->priv->gl_resource_lock); mix->priv->gl_resource_ready = FALSE; - if (mix->fbo) { - gst_gl_context_del_fbo (context, mix->fbo, mix->depthbuffer); - mix->fbo = 0; - mix->depthbuffer = 0; - } + if (mix->fbo) + gst_object_unref (mix->fbo); - if (!gst_gl_context_gen_fbo (context, out_width, out_height, - &mix->fbo, &mix->depthbuffer)) { + gst_gl_context_thread_add (context, + (GstGLContextThreadFunc) _mixer_create_fbo, mix); + if (!mix->fbo) { g_cond_signal (&mix->priv->gl_resource_cond); g_mutex_unlock (&mix->priv->gl_resource_lock); goto context_error; @@ -735,14 +738,13 @@ gst_gl_mixer_stop (GstAggregator * agg) { GstGLMixer *mix = GST_GL_MIXER (agg); GstGLMixerClass *mixer_class = GST_GL_MIXER_GET_CLASS (mix); - GstGLContext *context = GST_GL_BASE_MIXER (mix)->context; if (mixer_class->reset) mixer_class->reset (mix); + if (mix->fbo) { - gst_gl_context_del_fbo (context, mix->fbo, mix->depthbuffer); - mix->fbo = 0; - mix->depthbuffer = 0; + gst_object_unref (mix->fbo); + mix->fbo = NULL; } gst_gl_mixer_reset (mix); diff --git a/ext/gl/gstglmixer.h b/ext/gl/gstglmixer.h index de5a3dc4ef..309f6bbf0c 100644 --- a/ext/gl/gstglmixer.h +++ b/ext/gl/gstglmixer.h @@ -84,8 +84,7 @@ struct _GstGLMixer { GstGLBaseMixer vaggregator; - GLuint fbo; - GLuint depthbuffer; + GstGLFramebuffer *fbo; GstCaps *out_caps; diff --git a/ext/gl/gstglmosaic.c b/ext/gl/gstglmosaic.c index 701435f804..41441b0649 100644 --- a/ext/gl/gstglmosaic.c +++ b/ext/gl/gstglmosaic.c @@ -71,7 +71,7 @@ static gboolean gst_gl_mosaic_init_shader (GstGLMixer * mixer, static gboolean gst_gl_mosaic_process_textures (GstGLMixer * mixer, GstGLMemory * out_tex); -static void gst_gl_mosaic_callback (gpointer stuff); +static gboolean gst_gl_mosaic_callback (gpointer stuff); //vertex source static const gchar *mosaic_v_src = @@ -193,23 +193,31 @@ gst_gl_mosaic_init_shader (GstGLMixer * mixer, GstCaps * outcaps) mosaic_v_src, mosaic_f_src, &mosaic->shader); } +static void +_mosaic_render (GstGLContext * context, GstGLMosaic * mosaic) +{ + GstGLMixer *mixer = GST_GL_MIXER (mosaic); + + gst_gl_framebuffer_draw_to_texture (mixer->fbo, mosaic->out_tex, + gst_gl_mosaic_callback, mosaic); +} + static gboolean gst_gl_mosaic_process_textures (GstGLMixer * mix, GstGLMemory * out_tex) { GstGLMosaic *mosaic = GST_GL_MOSAIC (mix); + GstGLContext *context = GST_GL_BASE_MIXER (mix)->context; - //blocking call, use a FBO - gst_gl_context_use_fbo_v2 (GST_GL_BASE_MIXER (mix)->context, - GST_VIDEO_INFO_WIDTH (&GST_VIDEO_AGGREGATOR (mix)->info), - GST_VIDEO_INFO_HEIGHT (&GST_VIDEO_AGGREGATOR (mix)->info), mix->fbo, - mix->depthbuffer, out_tex->tex_id, gst_gl_mosaic_callback, - (gpointer) mosaic); + mosaic->out_tex = out_tex; + + gst_gl_context_thread_add (context, (GstGLContextThreadFunc) _mosaic_render, + mosaic); return TRUE; } /* opengl scene, params: input texture (not the output mixer->texture) */ -static void +static gboolean gst_gl_mosaic_callback (gpointer stuff) { GstGLMosaic *mosaic = GST_GL_MOSAIC (stuff); @@ -345,4 +353,6 @@ gst_gl_mosaic_callback (gpointer stuff) xrot += 0.6f; yrot += 0.4f; zrot += 0.8f; + + return TRUE; } diff --git a/ext/gl/gstglvideomixer.c b/ext/gl/gstglvideomixer.c index 176d27e547..4491823d13 100644 --- a/ext/gl/gstglvideomixer.c +++ b/ext/gl/gstglvideomixer.c @@ -475,7 +475,7 @@ static gboolean gst_gl_video_mixer_init_shader (GstGLMixer * mixer, static gboolean gst_gl_video_mixer_process_textures (GstGLMixer * mixer, GstGLMemory * out_tex); -static void gst_gl_video_mixer_callback (gpointer stuff); +static gboolean gst_gl_video_mixer_callback (gpointer stuff); /* *INDENT-OFF* */ @@ -1155,16 +1155,25 @@ gst_gl_video_mixer_init_shader (GstGLMixer * mixer, GstCaps * outcaps) video_mixer_f_src, &video_mixer->shader); } +static void +_video_mixer_process_gl (GstGLContext * context, GstGLVideoMixer * video_mixer) +{ + GstGLMixer *mixer = GST_GL_MIXER (video_mixer); + + gst_gl_framebuffer_draw_to_texture (mixer->fbo, video_mixer->out_tex, + gst_gl_video_mixer_callback, video_mixer); +} + static gboolean gst_gl_video_mixer_process_textures (GstGLMixer * mix, GstGLMemory * out_tex) { GstGLVideoMixer *video_mixer = GST_GL_VIDEO_MIXER (mix); + GstGLContext *context = GST_GL_BASE_MIXER (mix)->context; - gst_gl_context_use_fbo_v2 (GST_GL_BASE_MIXER (mix)->context, - GST_VIDEO_INFO_WIDTH (&GST_VIDEO_AGGREGATOR (mix)->info), - GST_VIDEO_INFO_HEIGHT (&GST_VIDEO_AGGREGATOR (mix)->info), - mix->fbo, mix->depthbuffer, - out_tex->tex_id, gst_gl_video_mixer_callback, (gpointer) video_mixer); + video_mixer->out_tex = out_tex; + + gst_gl_context_thread_add (context, + (GstGLContextThreadFunc) _video_mixer_process_gl, video_mixer); return TRUE; } @@ -1383,7 +1392,7 @@ _set_blend_state (GstGLVideoMixer * video_mixer, GstGLVideoMixerPad * mix_pad) } /* opengl scene, params: input texture (not the output mixer->texture) */ -static void +static gboolean gst_gl_video_mixer_callback (gpointer stuff) { GstGLVideoMixer *video_mixer = GST_GL_VIDEO_MIXER (stuff); @@ -1411,7 +1420,7 @@ gst_gl_video_mixer_callback (gpointer stuff) } if (!_draw_background (video_mixer)) - return; + return FALSE; gst_gl_shader_use (video_mixer->shader); @@ -1546,4 +1555,6 @@ gst_gl_video_mixer_callback (gpointer stuff) gl->Disable (GL_BLEND); gst_gl_context_clear_shader (GST_GL_BASE_MIXER (mixer)->context); + + return TRUE; } diff --git a/ext/gl/gstglvideomixer.h b/ext/gl/gstglvideomixer.h index a0776fd9fc..f3526465eb 100644 --- a/ext/gl/gstglvideomixer.h +++ b/ext/gl/gstglvideomixer.h @@ -125,6 +125,7 @@ struct _GstGLVideoMixer GLuint vao; GLuint vbo_indices; GLuint checker_vbo; + GstGLMemory *out_tex; }; struct _GstGLVideoMixerClass From d865d9126bc784d246c155d72d91a8ae6c192bce Mon Sep 17 00:00:00 2001 From: Nirbheek Chauhan Date: Fri, 12 Aug 2016 21:21:45 +0530 Subject: [PATCH 269/381] Add support for Meson as alternative/parallel build system MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit https://github.com/mesonbuild/meson With contributions from: Tim-Philipp Müller Matej Knopp Jussi Pakkanen (original port) Highlights of the features provided are: * Faster builds on Linux (~40-50% faster) * The ability to build with MSVC on Windows * Generate Visual Studio project files * Generate XCode project files * Much faster builds on Windows (on-par with Linux) * Seriously fast configure and building on embedded ... and many more. For more details see: http://blog.nirbheek.in/2016/05/gstreamer-and-meson-new-hope.html http://blog.nirbheek.in/2016/07/building-and-developing-gstreamer-using.html Building with Meson should work on both Linux and Windows, but may need a few more tweaks on other operating systems. --- gst/compositor/meson.build | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 gst/compositor/meson.build diff --git a/gst/compositor/meson.build b/gst/compositor/meson.build new file mode 100644 index 0000000000..2e948fcb0c --- /dev/null +++ b/gst/compositor/meson.build @@ -0,0 +1,33 @@ +compositor_sources = [ + 'blend.c', + 'compositor.c', +] + +orcsrc = 'compositororc' +if have_orcc + orc_h = custom_target(orcsrc + '.h', + input : orcsrc + '.orc', + output : orcsrc + '.h', + command : orcc_args + ['--header', '-o', '@OUTPUT@', '@INPUT@']) + orc_c = custom_target(orcsrc + '.c', + input : orcsrc + '.orc', + output : orcsrc + '.c', + command : orcc_args + ['--implementation', '-o', '@OUTPUT@', '@INPUT@']) +else + orc_h = configure_file(input : orcsrc + '-dist.h', + output : orcsrc + '.h', + configuration : configuration_data()) + orc_c = configure_file(input : orcsrc + '-dist.c', + output : orcsrc + '.c', + configuration : configuration_data()) +endif + +gstcompositor = library('gstcompositor', + compositor_sources, orc_c, orc_h, + c_args : gst_plugins_bad_args + ['-DGST_USE_UNSTABLE_API'], + include_directories : [configinc], + dependencies : [gstbadvideo_dep, gstbadbase_dep, gstvideo_dep, gstbase_dep, + orc_dep, libm], + install : true, + install_dir : plugins_install_dir, +) From 3010a28f07bd6985954cc3a0c043b6ac2a4d0bc0 Mon Sep 17 00:00:00 2001 From: Jan Schmidt Date: Fri, 26 Aug 2016 00:29:34 +1000 Subject: [PATCH 270/381] glstereomix: Fix caps negotiation The videoaggregator negotiation sequence changed some time back and broke glstereomix. Instead of doing nego incorrectly in the find_best_format() vfunc, do it directly in the update_caps() method. --- ext/gl/gstglstereomix.c | 201 +++++++++++++++++----------------------- ext/gl/gstglstereomix.h | 3 - 2 files changed, 86 insertions(+), 118 deletions(-) diff --git a/ext/gl/gstglstereomix.c b/ext/gl/gstglstereomix.c index 663c1b744a..1c214d8d66 100644 --- a/ext/gl/gstglstereomix.c +++ b/ext/gl/gstglstereomix.c @@ -106,11 +106,6 @@ static gboolean gst_gl_stereo_mix_start (GstAggregator * agg); static gboolean gst_gl_stereo_mix_src_query (GstAggregator * agg, GstQuery * query); -static void -gst_gl_stereo_mix_find_best_format (GstVideoAggregator * vagg, - GstCaps * downstream_caps, GstVideoInfo * best_info, - gboolean * at_least_one_alpha); - static void gst_gl_stereo_mix_set_property (GObject * object, guint prop_id, const GValue * value, GParamSpec * pspec); static void gst_gl_stereo_mix_get_property (GObject * object, guint prop_id, @@ -163,7 +158,6 @@ gst_gl_stereo_mix_class_init (GstGLStereoMixClass * klass) videoaggregator_class->negotiated_caps = _negotiated_caps; videoaggregator_class->get_output_buffer = gst_gl_stereo_mix_get_output_buffer; - videoaggregator_class->find_best_format = gst_gl_stereo_mix_find_best_format; base_mix_class->supported_gl_api = GST_GL_API_GLES2 | GST_GL_API_OPENGL | GST_GL_API_OPENGL3; @@ -438,13 +432,97 @@ get_converted_caps (GstGLStereoMix * mix, GstCaps * caps) return result; } -/* Return the possible output caps we decided in find_best_format() */ +/* Return the possible output caps based on inputs and downstream prefs */ static GstCaps * _update_caps (GstVideoAggregator * vagg, GstCaps * caps, GstCaps * filter) { GstGLStereoMix *mix = GST_GL_STEREO_MIX (vagg); + GList *l; + gint best_width = -1, best_height = -1; + gdouble best_fps = -1, cur_fps; + gint best_fps_n = 0, best_fps_d = 1; + GstVideoInfo *mix_info; + GstCaps *blend_caps, *tmp_caps; + GstCaps *out_caps; - return gst_caps_ref (mix->out_caps); + GST_OBJECT_LOCK (vagg); + + for (l = GST_ELEMENT (vagg)->sinkpads; l; l = l->next) { + GstVideoAggregatorPad *pad = l->data; + GstVideoInfo tmp = pad->info; + gint this_width, this_height; + gint fps_n, fps_d; + + if (!pad->info.finfo) + continue; + + /* This can happen if we release a pad and another pad hasn't been negotiated_caps yet */ + if (GST_VIDEO_INFO_FORMAT (&pad->info) == GST_VIDEO_FORMAT_UNKNOWN) + continue; + + /* Convert to per-view width/height for unpacked forms */ + gst_video_multiview_video_info_change_mode (&tmp, + GST_VIDEO_MULTIVIEW_MODE_SEPARATED, GST_VIDEO_MULTIVIEW_FLAGS_NONE); + + this_width = GST_VIDEO_INFO_WIDTH (&tmp); + this_height = GST_VIDEO_INFO_HEIGHT (&tmp); + fps_n = GST_VIDEO_INFO_FPS_N (&tmp); + fps_d = GST_VIDEO_INFO_FPS_D (&tmp); + + GST_INFO_OBJECT (vagg, "Input pad %" GST_PTR_FORMAT + " w %u h %u", pad, this_width, this_height); + + if (this_width == 0 || this_height == 0) + continue; + + if (best_width < this_width) + best_width = this_width; + if (best_height < this_height) + best_height = this_height; + + if (fps_d == 0) + cur_fps = 0.0; + else + gst_util_fraction_to_double (fps_n, fps_d, &cur_fps); + + if (best_fps < cur_fps) { + best_fps = cur_fps; + best_fps_n = fps_n; + best_fps_d = fps_d; + } + + /* FIXME: Preserve PAR for at least one input when different sized inputs */ + } + GST_OBJECT_UNLOCK (vagg); + + mix_info = &mix->mix_info; + gst_video_info_set_format (mix_info, GST_VIDEO_FORMAT_RGBA, best_width, + best_height); + + GST_VIDEO_INFO_FPS_N (mix_info) = best_fps_n; + GST_VIDEO_INFO_FPS_D (mix_info) = best_fps_d; + + GST_VIDEO_INFO_MULTIVIEW_MODE (mix_info) = GST_VIDEO_MULTIVIEW_MODE_SEPARATED; + GST_VIDEO_INFO_VIEWS (mix_info) = 2; + + /* FIXME: If input is marked as flipped or flopped, preserve those flags */ + GST_VIDEO_INFO_MULTIVIEW_FLAGS (mix_info) = GST_VIDEO_MULTIVIEW_FLAGS_NONE; + + /* Choose our output format based on downstream preferences */ + blend_caps = gst_video_info_to_caps (mix_info); + + gst_caps_set_features (blend_caps, 0, + gst_caps_features_from_string (GST_CAPS_FEATURE_MEMORY_GL_MEMORY)); + + tmp_caps = get_converted_caps (GST_GL_STEREO_MIX (vagg), blend_caps); + gst_caps_unref (blend_caps); + + out_caps = gst_caps_intersect (caps, tmp_caps); + gst_caps_unref (tmp_caps); + + GST_DEBUG_OBJECT (vagg, "Possible output caps %" GST_PTR_FORMAT, out_caps); + + return out_caps; } /* Called after videoaggregator fixates our caps */ @@ -462,8 +540,6 @@ _negotiated_caps (GstVideoAggregator * vagg, GstCaps * caps) return FALSE; /* Update the glview_convert output */ - if (!gst_video_info_from_caps (&mix->out_info, caps)) - return FALSE; /* We can configure the view_converter now */ gst_gl_view_convert_set_context (mix->viewconvert, @@ -585,108 +661,3 @@ gst_gl_stereo_mix_process_frames (GstGLStereoMix * mixer) return TRUE; } - -/* Iterate the input sink pads, and choose the blend format - * we will generate before output conversion, which is RGBA - * at some suitable size */ -static void -gst_gl_stereo_mix_find_best_format (GstVideoAggregator * vagg, - GstCaps * downstream_caps, GstVideoInfo * best_info, - gboolean * at_least_one_alpha) -{ - GstGLStereoMix *mix = GST_GL_STEREO_MIX (vagg); - GList *l; - gint best_width = -1, best_height = -1; - gdouble best_fps = -1, cur_fps; - gint best_fps_n = 0, best_fps_d = 1; - GstVideoInfo *mix_info; - GstCaps *blend_caps, *tmp_caps; - - /* We'll deal with alpha internally, so just tell aggregator to - * be quiet */ - *at_least_one_alpha = FALSE; - - GST_OBJECT_LOCK (vagg); - - for (l = GST_ELEMENT (vagg)->sinkpads; l; l = l->next) { - GstVideoAggregatorPad *pad = l->data; - GstVideoInfo tmp = pad->info; - gint this_width, this_height; - gint fps_n, fps_d; - - if (!pad->info.finfo) - continue; - - /* This can happen if we release a pad and another pad hasn't been negotiated_caps yet */ - if (GST_VIDEO_INFO_FORMAT (&pad->info) == GST_VIDEO_FORMAT_UNKNOWN) - continue; - - /* Convert to per-view width/height for unpacked forms */ - gst_video_multiview_video_info_change_mode (&tmp, - GST_VIDEO_MULTIVIEW_MODE_SEPARATED, GST_VIDEO_MULTIVIEW_FLAGS_NONE); - - this_width = GST_VIDEO_INFO_WIDTH (&tmp); - this_height = GST_VIDEO_INFO_HEIGHT (&tmp); - fps_n = GST_VIDEO_INFO_FPS_N (&tmp); - fps_d = GST_VIDEO_INFO_FPS_D (&tmp); - - GST_INFO_OBJECT (vagg, "Input pad %" GST_PTR_FORMAT - " w %u h %u", pad, this_width, this_height); - - if (this_width == 0 || this_height == 0) - continue; - - if (best_width < this_width) - best_width = this_width; - if (best_height < this_height) - best_height = this_height; - - if (fps_d == 0) - cur_fps = 0.0; - else - gst_util_fraction_to_double (fps_n, fps_d, &cur_fps); - - if (best_fps < cur_fps) { - best_fps = cur_fps; - best_fps_n = fps_n; - best_fps_d = fps_d; - } - - /* FIXME: Preserve PAR for at least one input when different sized inputs */ - } - GST_OBJECT_UNLOCK (vagg); - - mix_info = &mix->mix_info; - gst_video_info_set_format (mix_info, GST_VIDEO_FORMAT_RGBA, best_width, - best_height); - - GST_VIDEO_INFO_FPS_N (mix_info) = best_fps_n; - GST_VIDEO_INFO_FPS_D (mix_info) = best_fps_d; - - GST_VIDEO_INFO_MULTIVIEW_MODE (mix_info) = GST_VIDEO_MULTIVIEW_MODE_SEPARATED; - GST_VIDEO_INFO_VIEWS (mix_info) = 2; - - /* FIXME: If input is marked as flipped or flopped, preserve those flags */ - GST_VIDEO_INFO_MULTIVIEW_FLAGS (mix_info) = GST_VIDEO_MULTIVIEW_FLAGS_NONE; - - /* Choose our output format based on downstream preferences */ - blend_caps = gst_video_info_to_caps (mix_info); - - gst_caps_set_features (blend_caps, 0, - gst_caps_features_from_string (GST_CAPS_FEATURE_MEMORY_GL_MEMORY)); - - tmp_caps = get_converted_caps (GST_GL_STEREO_MIX (vagg), blend_caps); - gst_caps_unref (blend_caps); - - if (mix->out_caps) - gst_caps_unref (mix->out_caps); - - mix->out_caps = gst_caps_intersect (downstream_caps, tmp_caps); - gst_caps_unref (tmp_caps); - - GST_DEBUG_OBJECT (vagg, "Possible output caps %" GST_PTR_FORMAT, - mix->out_caps); - /* Tell videoaggregator our preferred size. Actual info gets - * overridden during caps nego */ - *best_info = *mix_info; -} diff --git a/ext/gl/gstglstereomix.h b/ext/gl/gstglstereomix.h index b0f1bd232c..9971a89101 100644 --- a/ext/gl/gstglstereomix.h +++ b/ext/gl/gstglstereomix.h @@ -69,9 +69,6 @@ struct _GstGLStereoMix GstGLViewConvert *viewconvert; GstGLStereoDownmix downmix_mode; - GstCaps *out_caps; - GstVideoInfo out_info; - GstVideoInfo mix_info; GPtrArray *input_frames; From 4e925aa85402328ba294b7c35ed433581dd17666 Mon Sep 17 00:00:00 2001 From: Jan Schmidt Date: Fri, 26 Aug 2016 02:06:00 +1000 Subject: [PATCH 271/381] gl: Add/update docs for glviewconvert, glstereomix Add some example pipelines for glstereomix, and fix up the example pipelines for glviewconvert --- ext/gl/gstglstereomix.c | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/ext/gl/gstglstereomix.c b/ext/gl/gstglstereomix.c index 1c214d8d66..18e7111a7f 100644 --- a/ext/gl/gstglstereomix.c +++ b/ext/gl/gstglstereomix.c @@ -21,6 +21,42 @@ * Boston, MA 02110-1301, USA. */ +/** + * SECTION:element-glstereomix + * + * Combine 2 input streams to produce a stereoscopic output + * stream. Input views are taken from the left pad and right pad + * respectively, and mixed according to their timelines. + * + * If either input stream is stereoscopic, the approproriate view + * (left or right) is taken from each stream and placed into the output. + * + * The multiview representation on the output is chosen according to + * the downstream caps. + * + * + * Examples + * |[ + * gst-launch-1.0 -v videotestsrc pattern=ball name=left \ + * videotestsrc name=right glstereomix name=mix \ + * left. ! vid/x-raw,width=640,height=480! glupload ! mix. \ + * right. ! video/x-raw,width=640,height=480! glupload ! mix. \ + * mix. ! video/x-raw'(memory:GLMemory)',multiview-mode=side-by-side ! \ + * queue ! glimagesink output-multiview-mode=side-by-side + * ]| Mix 2 different videotestsrc patterns into a side-by-side stereo image and display it. + * |[ + * gst-launch-1.0 -ev v4l2src name=left \ + * videotestsrc name=right \ + * glstereomix name=mix \ + * left. ! video/x-raw,width=640,height=480 ! glupload ! glcolorconvert ! mix. \ + * right. ! video/x-raw,width=640,height=480 ! glupload ! mix. \ + * mix. ! video/x-raw'(memory:GLMemory)',multiview-mode=top-bottom ! \ + * glcolorconvert ! gldownload ! queue ! x264enc ! h264parse ! \ + * mp4mux ! progressreport ! filesink location=output.mp4 + * ]| Mix the input from a camera to the left view, and videotestsrc to the right view, + * and encode as a top-bottom frame packed H.264 video. + * + */ #ifdef HAVE_CONFIG_H #include "config.h" #endif From e8a98ca5b3f5faf7fc20f53dcc246d06965dcbdf Mon Sep 17 00:00:00 2001 From: Jan Schmidt Date: Fri, 26 Aug 2016 02:07:27 +1000 Subject: [PATCH 272/381] gl: Update glvideomixer doc --- ext/gl/gstglvideomixer.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/ext/gl/gstglvideomixer.c b/ext/gl/gstglvideomixer.c index 4491823d13..04fab6f907 100644 --- a/ext/gl/gstglvideomixer.c +++ b/ext/gl/gstglvideomixer.c @@ -29,12 +29,12 @@ * Examples * |[ * gst-launch-1.0 glvideomixer name=m ! glimagesink \ - * videotestsrc ! video/x-raw, format=YUY2 ! m. \ - * videotestsrc pattern=12 ! video/x-raw, format=I420, framerate=5/1, width=100, height=200 ! queue ! m. \ - * videotestsrc ! video/x-raw, format=RGB, framerate=15/1, width=1500, height=1500 ! gleffects effect=3 ! queue ! m. \ - * videotestsrc ! gleffects effect=2 ! queue ! m. \ - * videotestsrc ! glfiltercube ! queue ! m. \ - * videotestsrc ! gleffects effect=6 ! queue ! m. + * videotestsrc ! video/x-raw, format=YUY2 ! glupload ! glcolorconvert ! m. \ + * videotestsrc pattern=12 ! video/x-raw, format=I420, framerate=5/1, width=100, height=200 ! queue ! \ + * glupload ! glcolorconvert ! m. \ + * videotestsrc ! glupload ! gleffects effect=2 ! queue ! m. \ + * videotestsrc ! glupload ! glfiltercube ! queue ! m. \ + * videotestsrc ! glupload ! gleffects effect=6 ! queue ! m.gst-launch-1.0 glvideomixer name=m ! glimagesink \ * ]| * */ From 4f537c636a31649910637a86c4a272865fcb3a07 Mon Sep 17 00:00:00 2001 From: Jan Schmidt Date: Fri, 26 Aug 2016 02:20:11 +1000 Subject: [PATCH 273/381] videoaggregator: Handle if update_caps() returns EMPTY Don't assume the returned caps from update_caps() is non-empty. --- gst-libs/gst/video/gstvideoaggregator.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index 1d40854d15..cd42d7ed2c 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -737,7 +737,8 @@ gst_video_aggregator_update_src_caps (GstVideoAggregator * vagg) GST_DEBUG_OBJECT (vagg, "updating caps from %" GST_PTR_FORMAT, downstream_caps); GST_DEBUG_OBJECT (vagg, " with filter %" GST_PTR_FORMAT, peercaps); - if (!(caps = vagg_klass->update_caps (vagg, downstream_caps, peercaps))) { + if (!(caps = vagg_klass->update_caps (vagg, downstream_caps, peercaps)) || + gst_caps_is_empty (caps)) { GST_WARNING_OBJECT (vagg, "Subclass failed to update provided caps"); gst_caps_unref (downstream_caps); if (peercaps) From d0a9cc0abcb52941daed1af1bf475ce231526cfc Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Wed, 5 Oct 2016 12:19:12 +1100 Subject: [PATCH 274/381] gl: GST_GL_TYPE -> GST_TYPE_GL Some deprecated symbols are kept for backwards compatibility --- ext/gl/gstglbasemixer.c | 10 +++++----- ext/gl/gstglvideomixer.c | 6 +++--- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/ext/gl/gstglbasemixer.c b/ext/gl/gstglbasemixer.c index 410a42022a..2611f3d9a9 100644 --- a/ext/gl/gstglbasemixer.c +++ b/ext/gl/gstglbasemixer.c @@ -157,7 +157,7 @@ _find_local_gl_context (GstGLBaseMixer * mix) gst_query_parse_context (query, &context); if (context) { s = gst_context_get_structure (context); - gst_structure_get (s, "context", GST_GL_TYPE_CONTEXT, &mix->context, + gst_structure_get (s, "context", GST_TYPE_GL_CONTEXT, &mix->context, NULL); } } @@ -166,7 +166,7 @@ _find_local_gl_context (GstGLBaseMixer * mix) gst_query_parse_context (query, &context); if (context) { s = gst_context_get_structure (context); - gst_structure_get (s, "context", GST_GL_TYPE_CONTEXT, &mix->context, + gst_structure_get (s, "context", GST_TYPE_GL_CONTEXT, &mix->context, NULL); } } @@ -317,7 +317,7 @@ gst_gl_base_mixer_sink_query (GstAggregator * agg, GstAggregatorPad * bpad, context = gst_context_new ("gst.gl.local_context", FALSE); s = gst_context_writable_structure (context); - gst_structure_set (s, "context", GST_GL_TYPE_CONTEXT, mix->context, + gst_structure_set (s, "context", GST_TYPE_GL_CONTEXT, mix->context, NULL); gst_query_set_context (query, context); gst_context_unref (context); @@ -416,7 +416,7 @@ gst_gl_base_mixer_class_init (GstGLBaseMixerClass * klass) g_param_spec_object ("context", "OpenGL context", "Get OpenGL context", - GST_GL_TYPE_CONTEXT, G_PARAM_READABLE | G_PARAM_STATIC_STRINGS)); + GST_TYPE_GL_CONTEXT, G_PARAM_READABLE | G_PARAM_STATIC_STRINGS)); /* Register the pad class */ g_type_class_ref (GST_TYPE_GL_BASE_MIXER_PAD); @@ -536,7 +536,7 @@ gst_gl_base_mixer_src_query (GstAggregator * agg, GstQuery * query) context = gst_context_new ("gst.gl.local_context", FALSE); s = gst_context_writable_structure (context); - gst_structure_set (s, "context", GST_GL_TYPE_CONTEXT, mix->context, + gst_structure_set (s, "context", GST_TYPE_GL_CONTEXT, mix->context, NULL); gst_query_set_context (query, context); gst_context_unref (context); diff --git a/ext/gl/gstglvideomixer.c b/ext/gl/gstglvideomixer.c index 04fab6f907..47d039678d 100644 --- a/ext/gl/gstglvideomixer.c +++ b/ext/gl/gstglvideomixer.c @@ -51,7 +51,7 @@ #define GST_CAT_DEFAULT gst_gl_video_mixer_debug GST_DEBUG_CATEGORY_STATIC (GST_CAT_DEFAULT); -#define GST_GL_TYPE_VIDEO_MIXER_BACKGROUND (gst_gl_video_mixer_background_get_type()) +#define GST_TYPE_GL_VIDEO_MIXER_BACKGROUND (gst_gl_video_mixer_background_get_type()) static GType gst_gl_video_mixer_background_get_type (void) { @@ -417,7 +417,7 @@ gst_gl_video_mixer_bin_class_init (GstGLVideoMixerBinClass * klass) g_object_class_install_property (gobject_class, PROP_BIN_BACKGROUND, g_param_spec_enum ("background", "Background", "Background type", - GST_GL_TYPE_VIDEO_MIXER_BACKGROUND, + GST_TYPE_GL_VIDEO_MIXER_BACKGROUND, DEFAULT_BACKGROUND, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); gst_element_class_set_metadata (element_class, "OpenGL video_mixer bin", @@ -875,7 +875,7 @@ gst_gl_video_mixer_class_init (GstGLVideoMixerClass * klass) g_object_class_install_property (gobject_class, PROP_BACKGROUND, g_param_spec_enum ("background", "Background", "Background type", - GST_GL_TYPE_VIDEO_MIXER_BACKGROUND, + GST_TYPE_GL_VIDEO_MIXER_BACKGROUND, DEFAULT_BACKGROUND, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); GST_GL_MIXER_CLASS (klass)->set_caps = gst_gl_video_mixer_init_shader; From af4de9cb42d83e99b97864fb5f4c69c289cca7ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Fri, 4 Nov 2016 16:26:50 +0200 Subject: [PATCH 275/381] gst: Fix more mentions of interlaced-mode to say interlace-mode --- tests/check/elements/compositor.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/check/elements/compositor.c b/tests/check/elements/compositor.c index e9f016543f..cfd9ceaefb 100644 --- a/tests/check/elements/compositor.c +++ b/tests/check/elements/compositor.c @@ -355,7 +355,7 @@ GST_START_TEST (test_caps_query) gst_caps_unref (caps); gst_caps_unref (restriction_caps); - /* check that compositor proxies downstream interlaced-mode */ + /* check that compositor proxies downstream interlace-mode */ restriction_caps = gst_caps_from_string ("video/x-raw, interlace-mode=(string)interleaved"); g_object_set (capsfilter, "caps", restriction_caps, NULL); @@ -412,7 +412,7 @@ GST_START_TEST (test_caps_query_interlaced) fail_unless (gst_caps_can_intersect (caps, caps_mixed)); gst_caps_unref (caps); - /* now set caps on the pad, it should restrict the interlaced-mode for + /* now set caps on the pad, it should restrict the interlace-mode for * future caps */ caps = gst_caps_from_string ("video/x-raw, width=100, height=100, " "format=RGB, framerate=1/1, interlace-mode=progressive"); From 0064e338a233133a2ddb23e00c25dd99d69242ed Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Thu, 10 Nov 2016 19:14:24 +1100 Subject: [PATCH 276/381] gl/utils: move gen_shader() to the plugin and remove del_shader() gst_gl_context_del_shader() can be replaced by a g_object_unref(). gst_gl_context_gen_shader() should be replaced by using GstGLSLStage. --- ext/gl/gstglmosaic.c | 4 ++-- ext/gl/gstglvideomixer.c | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/ext/gl/gstglmosaic.c b/ext/gl/gstglmosaic.c index 41441b0649..fcd4926a77 100644 --- a/ext/gl/gstglmosaic.c +++ b/ext/gl/gstglmosaic.c @@ -45,6 +45,7 @@ #endif #include "gstglmosaic.h" +#include "gstglutils.h" #define GST_CAT_DEFAULT gst_gl_mosaic_debug GST_DEBUG_CATEGORY_STATIC (GST_CAT_DEFAULT); @@ -177,8 +178,7 @@ gst_gl_mosaic_reset (GstGLMixer * mixer) //blocking call, wait the opengl thread has destroyed the shader if (mosaic->shader) - gst_gl_context_del_shader (GST_GL_BASE_MIXER (mixer)->context, - mosaic->shader); + gst_object_unref (mosaic->shader); mosaic->shader = NULL; } diff --git a/ext/gl/gstglvideomixer.c b/ext/gl/gstglvideomixer.c index 47d039678d..0cc4c677cb 100644 --- a/ext/gl/gstglvideomixer.c +++ b/ext/gl/gstglvideomixer.c @@ -47,6 +47,7 @@ #include "gstglvideomixer.h" #include "gstglmixerbin.h" +#include "gstglutils.h" #define GST_CAT_DEFAULT gst_gl_video_mixer_debug GST_DEBUG_CATEGORY_STATIC (GST_CAT_DEFAULT); @@ -1129,11 +1130,11 @@ gst_gl_video_mixer_reset (GstGLMixer * mixer) GST_DEBUG_OBJECT (mixer, "context:%p", context); if (video_mixer->shader) - gst_gl_context_del_shader (context, video_mixer->shader); + gst_object_unref (video_mixer->shader); video_mixer->shader = NULL; if (video_mixer->checker) - gst_gl_context_del_shader (context, video_mixer->checker); + gst_object_unref (video_mixer->checker); video_mixer->checker = NULL; if (GST_GL_BASE_MIXER (mixer)->context) @@ -1147,8 +1148,7 @@ gst_gl_video_mixer_init_shader (GstGLMixer * mixer, GstCaps * outcaps) GstGLVideoMixer *video_mixer = GST_GL_VIDEO_MIXER (mixer); if (video_mixer->shader) - gst_gl_context_del_shader (GST_GL_BASE_MIXER (mixer)->context, - video_mixer->shader); + gst_object_unref (video_mixer->shader); return gst_gl_context_gen_shader (GST_GL_BASE_MIXER (mixer)->context, gst_gl_shader_string_vertex_mat4_vertex_transform, From 8644352155f1d47d5a77e441cde5909fa8e49343 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Thu, 17 Nov 2016 20:20:15 +0200 Subject: [PATCH 277/381] videoaggregator: Mark pad as needing reconfiguration again if it failed And return FLUSHING instead of NOT_NEGOTIATED on flushing pads. https://bugzilla.gnome.org/show_bug.cgi?id=774623 --- gst-libs/gst/video/gstvideoaggregator.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index cd42d7ed2c..d1e4c90cc9 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -1452,7 +1452,8 @@ gst_video_aggregator_check_reconfigure (GstVideoAggregator * vagg, ret = gst_video_aggregator_update_src_caps (vagg); if (!ret) { - if (timeout && gst_pad_needs_reconfigure (GST_AGGREGATOR_SRC_PAD (vagg))) { + gst_pad_mark_reconfigure (GST_AGGREGATOR_SRC_PAD (vagg)); + if (timeout) { guint64 frame_duration; gint fps_d, fps_n; @@ -1482,7 +1483,10 @@ gst_video_aggregator_check_reconfigure (GstVideoAggregator * vagg, vagg->priv->nframes++; return GST_FLOW_NEEDS_DATA; } else { - return GST_FLOW_NOT_NEGOTIATED; + if (GST_PAD_IS_FLUSHING (GST_AGGREGATOR_SRC_PAD (vagg))) + return GST_FLOW_FLUSHING; + else + return GST_FLOW_NOT_NEGOTIATED; } } } From a9269d37c5204a6f7426b78037b1ca1e4b2dc971 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Tue, 13 Dec 2016 22:39:01 +0200 Subject: [PATCH 278/381] gst: Don't declare variables inside the for loop header This is a C99 feature. --- tests/check/elements/compositor.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/check/elements/compositor.c b/tests/check/elements/compositor.c index cfd9ceaefb..97211493d9 100644 --- a/tests/check/elements/compositor.c +++ b/tests/check/elements/compositor.c @@ -443,8 +443,9 @@ static void add_interlaced_mode_to_caps (GstCaps * caps, const gchar * mode) { GstStructure *s; + gint i; - for (gint i = 0; i < gst_caps_get_size (caps); i++) { + for (i = 0; i < gst_caps_get_size (caps); i++) { s = gst_caps_get_structure (caps, i); gst_structure_set (s, "interlace-mode", G_TYPE_STRING, mode, NULL); } From dd18a6eb45007867b95dae06896426681b2a8e16 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Wed, 14 Dec 2016 15:35:42 +0200 Subject: [PATCH 279/381] videoaggregator: Remove unlock() if set_info() fails There is not mutex locked here that needs to be unlocked. --- gst-libs/gst/video/gstvideoaggregator.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index d1e4c90cc9..ea2d10448b 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -796,8 +796,6 @@ gst_video_aggregator_update_src_caps (GstVideoAggregator * vagg) GstVideoAggregatorPad *pad = GST_VIDEO_AGGREGATOR_PAD (l->data); if (!vaggpad_klass->set_info (pad, vagg, &pad->info, &vagg->info)) { - GST_OBJECT_UNLOCK (vagg); - return FALSE; } } From 52fe0dbeaae838e99bf56a3606828e410cb913e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Wed, 14 Dec 2016 15:36:14 +0200 Subject: [PATCH 280/381] compositor: Reject multiview video Compositor does not support it currently and it needs special support for handling this correctly, and is rather non-trivial to implement for all formats. --- gst/compositor/compositor.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/gst/compositor/compositor.c b/gst/compositor/compositor.c index c9fd3fc1ea..2eac00d87a 100644 --- a/gst/compositor/compositor.c +++ b/gst/compositor/compositor.c @@ -283,6 +283,14 @@ gst_compositor_pad_set_info (GstVideoAggregatorPad * pad, cpad->convert = NULL; + if (GST_VIDEO_INFO_MULTIVIEW_MODE (current_info) != + GST_VIDEO_MULTIVIEW_MODE_NONE + && GST_VIDEO_INFO_MULTIVIEW_MODE (current_info) != + GST_VIDEO_MULTIVIEW_MODE_MONO) { + GST_FIXME_OBJECT (pad, "Multiview support is not implemented yet"); + return FALSE; + } + colorimetry = gst_video_colorimetry_to_string (&(current_info->colorimetry)); chroma = gst_video_chroma_to_string (current_info->chroma_site); From 0f4ed7e2d66442dd7f73be650c42ed98435c9837 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Wed, 14 Dec 2016 15:53:41 +0200 Subject: [PATCH 281/381] glvideomixer: Reject multiview video glvideomixer does not support it currently and it needs special support for handling this correctly, and is rather non-trivial to implement for all formats. --- ext/gl/gstglvideomixer.c | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/ext/gl/gstglvideomixer.c b/ext/gl/gstglvideomixer.c index 0cc4c677cb..ffb43d4e82 100644 --- a/ext/gl/gstglvideomixer.c +++ b/ext/gl/gstglvideomixer.c @@ -998,6 +998,30 @@ static GstCaps * _update_caps (GstVideoAggregator * vagg, GstCaps * caps, GstCaps * filter) { GstCaps *ret; + GList *l; + + GST_OBJECT_LOCK (vagg); + for (l = GST_ELEMENT (vagg)->sinkpads; l; l = l->next) { + GstVideoAggregatorPad *vaggpad = l->data; + + if (!vaggpad->info.finfo) + continue; + + if (GST_VIDEO_INFO_FORMAT (&vaggpad->info) == GST_VIDEO_FORMAT_UNKNOWN) + continue; + + if (GST_VIDEO_INFO_MULTIVIEW_MODE (&vaggpad->info) != + GST_VIDEO_MULTIVIEW_MODE_NONE + && GST_VIDEO_INFO_MULTIVIEW_MODE (&vaggpad->info) != + GST_VIDEO_MULTIVIEW_MODE_MONO) { + GST_FIXME_OBJECT (vaggpad, "Multiview support is not implemented yet"); + GST_OBJECT_UNLOCK (vagg); + return NULL; + } + + } + + GST_OBJECT_UNLOCK (vagg); if (filter) { ret = gst_caps_intersect (caps, filter); From e8ec52b8ef8ea29a35f2aa24807b563ef9c9b193 Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Tue, 10 Jan 2017 12:51:51 +1100 Subject: [PATCH 282/381] gl: remove custom control binding proxy Use the existing GstProxyControlBinding instead. --- ext/gl/gstglvideomixer.c | 55 ++++++++++++++++------------------------ 1 file changed, 22 insertions(+), 33 deletions(-) diff --git a/ext/gl/gstglvideomixer.c b/ext/gl/gstglvideomixer.c index ffb43d4e82..1c8655792a 100644 --- a/ext/gl/gstglvideomixer.c +++ b/ext/gl/gstglvideomixer.c @@ -44,6 +44,7 @@ #endif #include +#include #include "gstglvideomixer.h" #include "gstglmixerbin.h" @@ -338,39 +339,27 @@ _create_video_mixer_input (GstGLMixerBin * self, GstPad * mixer_pad) gst_object_unref (input); return NULL; } - - gst_gl_object_add_control_binding_proxy (GST_OBJECT (mixer_pad), - GST_OBJECT (input), "zorder"); - gst_gl_object_add_control_binding_proxy (GST_OBJECT (mixer_pad), - GST_OBJECT (input), "xpos"); - gst_gl_object_add_control_binding_proxy (GST_OBJECT (mixer_pad), - GST_OBJECT (input), "ypos"); - gst_gl_object_add_control_binding_proxy (GST_OBJECT (mixer_pad), - GST_OBJECT (input), "width"); - gst_gl_object_add_control_binding_proxy (GST_OBJECT (mixer_pad), - GST_OBJECT (input), "height"); - gst_gl_object_add_control_binding_proxy (GST_OBJECT (mixer_pad), - GST_OBJECT (input), "alpha"); - gst_gl_object_add_control_binding_proxy (GST_OBJECT (mixer_pad), - GST_OBJECT (input), "blend-equation-rgb"); - gst_gl_object_add_control_binding_proxy (GST_OBJECT (mixer_pad), - GST_OBJECT (input), "blend-equation-alpha"); - gst_gl_object_add_control_binding_proxy (GST_OBJECT (mixer_pad), - GST_OBJECT (input), "blend-function-src-rgb"); - gst_gl_object_add_control_binding_proxy (GST_OBJECT (mixer_pad), - GST_OBJECT (input), "blend-function-src-alpha"); - gst_gl_object_add_control_binding_proxy (GST_OBJECT (mixer_pad), - GST_OBJECT (input), "blend-function-dst-rgb"); - gst_gl_object_add_control_binding_proxy (GST_OBJECT (mixer_pad), - GST_OBJECT (input), "blend-function-dst-alpha"); - gst_gl_object_add_control_binding_proxy (GST_OBJECT (mixer_pad), - GST_OBJECT (input), "blend-constant-color-red"); - gst_gl_object_add_control_binding_proxy (GST_OBJECT (mixer_pad), - GST_OBJECT (input), "blend-constant-color-green"); - gst_gl_object_add_control_binding_proxy (GST_OBJECT (mixer_pad), - GST_OBJECT (input), "blend-constant-color-blue"); - gst_gl_object_add_control_binding_proxy (GST_OBJECT (mixer_pad), - GST_OBJECT (input), "blend-constant-color-alpha"); +#define ADD_BINDING(obj,ref,prop) \ + gst_object_add_control_binding (GST_OBJECT (obj), \ + gst_proxy_control_binding_new (GST_OBJECT (obj), prop, \ + GST_OBJECT (ref), prop)); + ADD_BINDING (mixer_pad, input, "zorder"); + ADD_BINDING (mixer_pad, input, "xpos"); + ADD_BINDING (mixer_pad, input, "ypos"); + ADD_BINDING (mixer_pad, input, "width"); + ADD_BINDING (mixer_pad, input, "height"); + ADD_BINDING (mixer_pad, input, "alpha"); + ADD_BINDING (mixer_pad, input, "blend-equation-rgb"); + ADD_BINDING (mixer_pad, input, "blend-equation-alpha"); + ADD_BINDING (mixer_pad, input, "blend-function-src-rgb"); + ADD_BINDING (mixer_pad, input, "blend-function-src-alpha"); + ADD_BINDING (mixer_pad, input, "blend-function-dst-rgb"); + ADD_BINDING (mixer_pad, input, "blend-function-dst-alpha"); + ADD_BINDING (mixer_pad, input, "blend-constant-color-red"); + ADD_BINDING (mixer_pad, input, "blend-constant-color-green"); + ADD_BINDING (mixer_pad, input, "blend-constant-color-blue"); + ADD_BINDING (mixer_pad, input, "blend-constant-color-alpha"); +#undef ADD_BINDING input->mixer_pad = mixer_pad; From b4301e2235aeee16399d1b3ac4a040ed086db238 Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Thu, 12 Jan 2017 01:57:29 +1100 Subject: [PATCH 283/381] glutils: remove trivial helper function gst_gl_caps_replace_all_caps_features() is only used in two places and can be trivially reproduced. --- ext/gl/gstglmixer.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/ext/gl/gstglmixer.c b/ext/gl/gstglmixer.c index 3fc5aa6f50..9a7446ab0b 100644 --- a/ext/gl/gstglmixer.c +++ b/ext/gl/gstglmixer.c @@ -220,15 +220,21 @@ static GstCaps * _update_caps (GstVideoAggregator * vagg, GstCaps * caps, GstCaps * filter) { GstCaps *tmp; + guint i, n; if (filter) { tmp = gst_caps_intersect (caps, filter); } else { - tmp = caps; + tmp = gst_caps_copy (caps); } - return gst_gl_caps_replace_all_caps_features (tmp, - GST_CAPS_FEATURE_MEMORY_GL_MEMORY); + n = gst_caps_get_size (tmp); + for (i = 0; i < n; i++) { + gst_caps_set_features (tmp, i, + gst_caps_features_from_string (GST_CAPS_FEATURE_MEMORY_GL_MEMORY)); + } + + return tmp; } static GstCaps * From 7fcbfe1234803832585d693c017d298c4e182c66 Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Thu, 12 Jan 2017 21:35:25 +1100 Subject: [PATCH 284/381] gl/utils: also take care of the local GL context in query functions Simplifies a deduplicates a lot of code in elements retrieving/setting the local OpenGL context. --- ext/gl/gstglbasemixer.c | 109 ++++------------------------------------ 1 file changed, 10 insertions(+), 99 deletions(-) diff --git a/ext/gl/gstglbasemixer.c b/ext/gl/gstglbasemixer.c index 2611f3d9a9..6e6e9493e0 100644 --- a/ext/gl/gstglbasemixer.c +++ b/ext/gl/gstglbasemixer.c @@ -145,39 +145,12 @@ gst_gl_base_mixer_sink_event (GstAggregator * agg, GstAggregatorPad * bpad, static gboolean _find_local_gl_context (GstGLBaseMixer * mix) { - GstQuery *query; - GstContext *context; - const GstStructure *s; - - if (mix->context) + if (gst_gl_query_local_gl_context (GST_ELEMENT (mix), GST_PAD_SRC, + &mix->context)) return TRUE; - - query = gst_query_new_context ("gst.gl.local_context"); - if (!mix->context && gst_gl_run_query (GST_ELEMENT (mix), query, GST_PAD_SRC)) { - gst_query_parse_context (query, &context); - if (context) { - s = gst_context_get_structure (context); - gst_structure_get (s, "context", GST_TYPE_GL_CONTEXT, &mix->context, - NULL); - } - } - if (!mix->context - && gst_gl_run_query (GST_ELEMENT (mix), query, GST_PAD_SINK)) { - gst_query_parse_context (query, &context); - if (context) { - s = gst_context_get_structure (context); - gst_structure_get (s, "context", GST_TYPE_GL_CONTEXT, &mix->context, - NULL); - } - } - - GST_DEBUG_OBJECT (mix, "found local context %p", mix->context); - - gst_query_unref (query); - - if (mix->context) + if (gst_gl_query_local_gl_context (GST_ELEMENT (mix), GST_PAD_SINK, + &mix->context)) return TRUE; - return FALSE; } @@ -296,39 +269,9 @@ gst_gl_base_mixer_sink_query (GstAggregator * agg, GstAggregatorPad * bpad, } case GST_QUERY_CONTEXT: { - const gchar *context_type; - GstContext *context, *old_context; - - ret = gst_gl_handle_context_query ((GstElement *) mix, query, - &mix->display, &mix->priv->other_context); - if (mix->display) - gst_gl_display_filter_gl_api (mix->display, - mix_class->supported_gl_api); - gst_query_parse_context_type (query, &context_type); - - if (g_strcmp0 (context_type, "gst.gl.local_context") == 0) { - GstStructure *s; - - gst_query_parse_context (query, &old_context); - - if (old_context) - context = gst_context_copy (old_context); - else - context = gst_context_new ("gst.gl.local_context", FALSE); - - s = gst_context_writable_structure (context); - gst_structure_set (s, "context", GST_TYPE_GL_CONTEXT, mix->context, - NULL); - gst_query_set_context (query, context); - gst_context_unref (context); - - ret = mix->context != NULL; - } - GST_LOG_OBJECT (mix, "context query of type %s %i", context_type, ret); - - if (ret) - return ret; - + if (gst_gl_handle_context_query ((GstElement *) mix, query, + mix->display, mix->context, mix->priv->other_context)) + return TRUE; break; } default: @@ -508,46 +451,14 @@ gst_gl_base_mixer_src_activate_mode (GstAggregator * aggregator, static gboolean gst_gl_base_mixer_src_query (GstAggregator * agg, GstQuery * query) { - gboolean res = FALSE; GstGLBaseMixer *mix = GST_GL_BASE_MIXER (agg); - GstGLBaseMixerClass *mix_class = GST_GL_BASE_MIXER_GET_CLASS (mix); switch (GST_QUERY_TYPE (query)) { case GST_QUERY_CONTEXT: { - const gchar *context_type; - GstContext *context, *old_context; - - res = gst_gl_handle_context_query ((GstElement *) mix, query, - &mix->display, &mix->priv->other_context); - if (mix->display) - gst_gl_display_filter_gl_api (mix->display, - mix_class->supported_gl_api); - gst_query_parse_context_type (query, &context_type); - - if (g_strcmp0 (context_type, "gst.gl.local_context") == 0) { - GstStructure *s; - - gst_query_parse_context (query, &old_context); - - if (old_context) - context = gst_context_copy (old_context); - else - context = gst_context_new ("gst.gl.local_context", FALSE); - - s = gst_context_writable_structure (context); - gst_structure_set (s, "context", GST_TYPE_GL_CONTEXT, mix->context, - NULL); - gst_query_set_context (query, context); - gst_context_unref (context); - - res = mix->context != NULL; - } - GST_LOG_OBJECT (mix, "context query of type %s %i", context_type, res); - - if (res) - return res; - + if (gst_gl_handle_context_query ((GstElement *) mix, query, + mix->display, mix->context, mix->priv->other_context)) + return TRUE; break; } default: From c6bf4b36e6fad65f3ce48b92f0e8910be42c4977 Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Wed, 18 Jan 2017 15:03:48 +1100 Subject: [PATCH 285/381] glmixer: ensure caps are writable after intersection gst_caps_intersect () may return an increased reference of one of the input caps. Fixes critical in the simple-launch-lines test: Unexpected critical/warning: gst_caps_set_features: assertion 'IS_WRITABLE (caps)' failed --- ext/gl/gstglmixer.c | 1 + 1 file changed, 1 insertion(+) diff --git a/ext/gl/gstglmixer.c b/ext/gl/gstglmixer.c index 9a7446ab0b..b8f4da0267 100644 --- a/ext/gl/gstglmixer.c +++ b/ext/gl/gstglmixer.c @@ -224,6 +224,7 @@ _update_caps (GstVideoAggregator * vagg, GstCaps * caps, GstCaps * filter) if (filter) { tmp = gst_caps_intersect (caps, filter); + tmp = gst_caps_make_writable (tmp); } else { tmp = gst_caps_copy (caps); } From 12540146eacc9d680e22ab4b7e5cd96ce875eada Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Tue, 28 Feb 2017 10:53:04 +0200 Subject: [PATCH 286/381] gl: Rename gst_gl_get_affine_transformation_meta_as_ndc_ext() to prevent symbol conflict The same symbol also exists in libgstgl, although marked as private and internal. This has no effect when doing static linking and there's a symbol conflict. --- ext/gl/gstglvideomixer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/gl/gstglvideomixer.c b/ext/gl/gstglvideomixer.c index 1c8655792a..b9ea63ef54 100644 --- a/ext/gl/gstglvideomixer.c +++ b/ext/gl/gstglvideomixer.c @@ -1535,7 +1535,7 @@ gst_gl_video_mixer_callback (gpointer stuff) af_meta = gst_buffer_get_video_affine_transformation_meta (vagg_pad->buffer); - gst_gl_get_affine_transformation_meta_as_ndc (af_meta, matrix); + gst_gl_get_affine_transformation_meta_as_ndc_ext (af_meta, matrix); gst_gl_shader_set_uniform_matrix_4fv (video_mixer->shader, "u_transformation", 1, FALSE, matrix); } From aee020c26f1f1ae1b16e95bc6212ad19644765d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Tue, 28 Feb 2017 13:06:41 +0200 Subject: [PATCH 287/381] glvideomixer/compositor: Correctly error out if calculating DAR fails CID 1320700 --- ext/gl/gstglvideomixer.c | 16 +++++++++------- gst/compositor/compositor.c | 1 + 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/ext/gl/gstglvideomixer.c b/ext/gl/gstglvideomixer.c index b9ea63ef54..ebd6827524 100644 --- a/ext/gl/gstglvideomixer.c +++ b/ext/gl/gstglvideomixer.c @@ -962,9 +962,13 @@ _mixer_pad_get_output_size (GstGLVideoMixer * mix, mix_pad->height <= 0 ? GST_VIDEO_INFO_HEIGHT (&vagg_pad->info) : mix_pad->height; - gst_video_calculate_display_ratio (&dar_n, &dar_d, pad_width, pad_height, - GST_VIDEO_INFO_PAR_N (&vagg_pad->info), - GST_VIDEO_INFO_PAR_D (&vagg_pad->info), out_par_n, out_par_d); + if (!gst_video_calculate_display_ratio (&dar_n, &dar_d, pad_width, pad_height, + GST_VIDEO_INFO_PAR_N (&vagg_pad->info), + GST_VIDEO_INFO_PAR_D (&vagg_pad->info), out_par_n, out_par_d)) { + GST_WARNING_OBJECT (mix_pad, "Cannot calculate display aspect ratio"); + *width = *height = 0; + return; + } GST_LOG_OBJECT (mix_pad, "scaling %ux%u by %u/%u (%u/%u / %u/%u)", pad_width, pad_height, dar_n, dar_d, GST_VIDEO_INFO_PAR_N (&vagg_pad->info), GST_VIDEO_INFO_PAR_D (&vagg_pad->info), out_par_n, out_par_d); @@ -977,10 +981,8 @@ _mixer_pad_get_output_size (GstGLVideoMixer * mix, pad_width = gst_util_uint64_scale_int (pad_height, dar_n, dar_d); } - if (width) - *width = pad_width; - if (height) - *height = pad_height; + *width = pad_width; + *height = pad_height; } static GstCaps * diff --git a/gst/compositor/compositor.c b/gst/compositor/compositor.c index 2eac00d87a..068d2bb721 100644 --- a/gst/compositor/compositor.c +++ b/gst/compositor/compositor.c @@ -244,6 +244,7 @@ _mixer_pad_get_output_size (GstCompositor * comp, GST_VIDEO_INFO_PAR_D (&vagg_pad->info), out_par_n, out_par_d)) { GST_WARNING_OBJECT (comp_pad, "Cannot calculate display aspect ratio"); *width = *height = 0; + return; } GST_LOG_OBJECT (comp_pad, "scaling %ux%u by %u/%u (%u/%u / %u/%u)", pad_width, pad_height, dar_n, dar_d, GST_VIDEO_INFO_PAR_N (&vagg_pad->info), From 7ac883cebfe9828f94c5693c55247065cc61c4f6 Mon Sep 17 00:00:00 2001 From: George Kiagiadakis Date: Fri, 3 Mar 2017 16:20:15 +0200 Subject: [PATCH 288/381] videoaggregator: redo src caps negotiation if a sink pad's caps have changed in the meantime https://bugzilla.gnome.org/show_bug.cgi?id=755782 --- gst-libs/gst/video/gstvideoaggregator.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index ea2d10448b..2abb4d569b 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -1448,6 +1448,7 @@ gst_video_aggregator_check_reconfigure (GstVideoAggregator * vagg, || gst_pad_check_reconfigure (GST_AGGREGATOR_SRC_PAD (vagg))) { gboolean ret; + restart: ret = gst_video_aggregator_update_src_caps (vagg); if (!ret) { gst_pad_mark_reconfigure (GST_AGGREGATOR_SRC_PAD (vagg)); @@ -1486,6 +1487,13 @@ gst_video_aggregator_check_reconfigure (GstVideoAggregator * vagg, else return GST_FLOW_NOT_NEGOTIATED; } + } else { + /* It is possible that during gst_video_aggregator_update_src_caps() + * we got a caps change on one of the sink pads, in which case we need + * to redo the negotiation + * - https://bugzilla.gnome.org/show_bug.cgi?id=755782 */ + if (gst_pad_check_reconfigure (GST_AGGREGATOR_SRC_PAD (vagg))) + goto restart; } } From a7c46e4a08cf1134f67dd4dfb5e416394ed4c524 Mon Sep 17 00:00:00 2001 From: Mathieu Duponchelle Date: Fri, 7 Apr 2017 18:49:52 +0200 Subject: [PATCH 289/381] videoaggregator: Do not mix the same buffer twice when EOS. When entering this code path, we know that: We received EOS on this pad. We consumed all its buffers. In any case, we want to replace vaggpad->buffer with NULL, otherwise we will end up mixing the same buffer twice. https://bugzilla.gnome.org/show_bug.cgi?id=781037 --- gst-libs/gst/video/gstvideoaggregator.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index 2abb4d569b..e17c111d45 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -1268,7 +1268,9 @@ gst_video_aggregator_fill_queues (GstVideoAggregator * vagg, } else if (is_eos) { eos = FALSE; } - } else if (is_eos) { + } + + if (is_eos) { gst_buffer_replace (&pad->buffer, NULL); } } From 2bf26969450b7a52031d17fd274b38f39fdfca3e Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Wed, 8 Mar 2017 15:01:13 -0300 Subject: [PATCH 290/381] docs: Port all docstring to gtk-doc markdown --- ext/gl/gstglmosaic.c | 6 ++-- ext/gl/gstglstereomix.c | 8 ++--- ext/gl/gstglvideomixer.c | 6 ++-- gst-libs/gst/video/gstvideoaggregator.c | 1 + gst/compositor/compositor.c | 44 +++++++------------------ 5 files changed, 23 insertions(+), 42 deletions(-) diff --git a/ext/gl/gstglmosaic.c b/ext/gl/gstglmosaic.c index fcd4926a77..77aec599cd 100644 --- a/ext/gl/gstglmosaic.c +++ b/ext/gl/gstglmosaic.c @@ -20,14 +20,14 @@ /** * SECTION:element-glmosaic + * @title: glmosaic * * glmixer sub element. N gl sink pads to 1 source pad. * N + 1 OpenGL contexts shared together. * N <= 6 because the rendering is more a like a cube than a mosaic * Each opengl input stream is rendered on a cube face * - * - * Examples + * ## Examples * |[ * gst-launch-1.0 videotestsrc ! video/x-raw, format=YUY2 ! queue ! glmosaic name=m ! glimagesink \ * videotestsrc pattern=12 ! video/x-raw, format=I420, framerate=5/1, width=100, height=200 ! queue ! m. \ @@ -37,7 +37,7 @@ * videotestsrc ! gleffects effect=6 ! queue ! m. * ]| * FBO (Frame Buffer Object) is required. - * + * */ #ifdef HAVE_CONFIG_H diff --git a/ext/gl/gstglstereomix.c b/ext/gl/gstglstereomix.c index 18e7111a7f..78a2934fad 100644 --- a/ext/gl/gstglstereomix.c +++ b/ext/gl/gstglstereomix.c @@ -23,6 +23,7 @@ /** * SECTION:element-glstereomix + * @title: glstereomix * * Combine 2 input streams to produce a stereoscopic output * stream. Input views are taken from the left pad and right pad @@ -34,8 +35,7 @@ * The multiview representation on the output is chosen according to * the downstream caps. * - * - * Examples + * ## Examples * |[ * gst-launch-1.0 -v videotestsrc pattern=ball name=left \ * videotestsrc name=right glstereomix name=mix \ @@ -52,10 +52,10 @@ * right. ! video/x-raw,width=640,height=480 ! glupload ! mix. \ * mix. ! video/x-raw'(memory:GLMemory)',multiview-mode=top-bottom ! \ * glcolorconvert ! gldownload ! queue ! x264enc ! h264parse ! \ - * mp4mux ! progressreport ! filesink location=output.mp4 + * mp4mux ! progressreport ! filesink location=output.mp4 * ]| Mix the input from a camera to the left view, and videotestsrc to the right view, * and encode as a top-bottom frame packed H.264 video. - * + * */ #ifdef HAVE_CONFIG_H #include "config.h" diff --git a/ext/gl/gstglvideomixer.c b/ext/gl/gstglvideomixer.c index ebd6827524..5df126d878 100644 --- a/ext/gl/gstglvideomixer.c +++ b/ext/gl/gstglvideomixer.c @@ -20,13 +20,13 @@ /** * SECTION:element-glvideomixer + * @title: glvideomixer * * Composites a number of streams into a single output scene using OpenGL in * a similar fashion to compositor and videomixer. See the compositor plugin * for documentation about the #GstGLVideoMixerPad properties. * - * - * Examples + * ## Examples * |[ * gst-launch-1.0 glvideomixer name=m ! glimagesink \ * videotestsrc ! video/x-raw, format=YUY2 ! glupload ! glcolorconvert ! m. \ @@ -36,7 +36,7 @@ * videotestsrc ! glupload ! glfiltercube ! queue ! m. \ * videotestsrc ! glupload ! gleffects effect=6 ! queue ! m.gst-launch-1.0 glvideomixer name=m ! glimagesink \ * ]| - * + * */ #ifdef HAVE_CONFIG_H diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index e17c111d45..1616b27244 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -20,6 +20,7 @@ /** * SECTION:gstvideoaggregator + * @title: GstVideoAggregator * @short_description: Base class for video aggregators * * VideoAggregator can accept AYUV, ARGB and BGRA video streams. For each of the requested diff --git a/gst/compositor/compositor.c b/gst/compositor/compositor.c index 068d2bb721..fab75c6394 100644 --- a/gst/compositor/compositor.c +++ b/gst/compositor/compositor.c @@ -22,6 +22,7 @@ /** * SECTION:element-compositor + * @title: compositor * * Compositor can accept AYUV, ARGB and BGRA video streams. For each of the requested * sink pads it will compare the incoming geometry and framerate to define the @@ -29,40 +30,19 @@ * biggest incoming video stream and the framerate of the fastest incoming one. * * Compositor will do colorspace conversion. - * + * * Individual parameters for each input stream can be configured on the * #GstCompositorPad: * - * - * - * "xpos": The x-coordinate position of the top-left corner of the picture - * (#gint) - * - * - * "ypos": The y-coordinate position of the top-left corner of the picture - * (#gint) - * - * - * "width": The width of the picture; the input will be scaled if necessary - * (#gint) - * - * - * "height": The height of the picture; the input will be scaled if necessary - * (#gint) - * - * - * "alpha": The transparency of the picture; between 0.0 and 1.0. The blending - * is a simple copy when fully-transparent (0.0) and fully-opaque (1.0). - * (#gdouble) - * - * - * "zorder": The z-order position of the picture in the composition - * (#guint) - * - * + * * "xpos": The x-coordinate position of the top-left corner of the picture (#gint) + * * "ypos": The y-coordinate position of the top-left corner of the picture (#gint) + * * "width": The width of the picture; the input will be scaled if necessary (#gint) + * * "height": The height of the picture; the input will be scaled if necessary (#gint) + * * "alpha": The transparency of the picture; between 0.0 and 1.0. The blending + * is a simple copy when fully-transparent (0.0) and fully-opaque (1.0). (#gdouble) + * * "zorder": The z-order position of the picture in the composition (#guint) * - * - * Sample pipelines + * ## Sample pipelines * |[ * gst-launch-1.0 \ * videotestsrc pattern=1 ! \ @@ -85,7 +65,7 @@ * compositor name=comp ! videoconvert ! ximagesink \ * videotestsrc ! \ * video/x-raw, framerate=\(fraction\)5/1, width=320, height=240 ! comp. - * ]| A pipeline to demostrate bgra comping. (This does not demonstrate alpha blending). + * ]| A pipeline to demostrate bgra comping. (This does not demonstrate alpha blending). * |[ * gst-launch-1.0 videotestsrc pattern=1 ! \ * video/x-raw,format =I420, framerate=\(fraction\)10/1, width=100, height=100 ! \ @@ -103,7 +83,7 @@ * "video/x-raw,format=AYUV,width=800,height=600,framerate=(fraction)10/1" ! \ * timeoverlay ! queue2 ! comp. * ]| A pipeline to demonstrate synchronized compositing (the second stream starts after 3 seconds) - * + * */ #ifdef HAVE_CONFIG_H From 449a1b391bcb3f2bd67bb0a94872bc3a4333411e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim-Philipp=20M=C3=BCller?= Date: Sun, 7 May 2017 11:41:06 +0100 Subject: [PATCH 291/381] glvideomixer: remove extraneous \ from example launch line in docs --- ext/gl/gstglvideomixer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/gl/gstglvideomixer.c b/ext/gl/gstglvideomixer.c index 5df126d878..76e6716781 100644 --- a/ext/gl/gstglvideomixer.c +++ b/ext/gl/gstglvideomixer.c @@ -34,7 +34,7 @@ * glupload ! glcolorconvert ! m. \ * videotestsrc ! glupload ! gleffects effect=2 ! queue ! m. \ * videotestsrc ! glupload ! glfiltercube ! queue ! m. \ - * videotestsrc ! glupload ! gleffects effect=6 ! queue ! m.gst-launch-1.0 glvideomixer name=m ! glimagesink \ + * videotestsrc ! glupload ! gleffects effect=6 ! queue ! m.gst-launch-1.0 glvideomixer name=m ! glimagesink * ]| * */ From a82c4e1fed19cc585461da7bfe0a71a65bd65a22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim-Philipp=20M=C3=BCller?= Date: Sun, 7 May 2017 11:47:40 +0100 Subject: [PATCH 292/381] glvideomixer: fix whole example launch line actually --- ext/gl/gstglvideomixer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/gl/gstglvideomixer.c b/ext/gl/gstglvideomixer.c index 76e6716781..06bdf3e102 100644 --- a/ext/gl/gstglvideomixer.c +++ b/ext/gl/gstglvideomixer.c @@ -34,7 +34,7 @@ * glupload ! glcolorconvert ! m. \ * videotestsrc ! glupload ! gleffects effect=2 ! queue ! m. \ * videotestsrc ! glupload ! glfiltercube ! queue ! m. \ - * videotestsrc ! glupload ! gleffects effect=6 ! queue ! m.gst-launch-1.0 glvideomixer name=m ! glimagesink + * videotestsrc ! glupload ! gleffects effect=6 ! queue ! m. * ]| * */ From c483cbf9046fc5b3911403146e550e3394eaa52c Mon Sep 17 00:00:00 2001 From: Nicolas Dufresne Date: Tue, 16 May 2017 14:05:52 -0400 Subject: [PATCH 293/381] Remove plugin specific static build option Static and dynamic plugins now have the same interface. The standard --enable-static/--enable-shared toggle are sufficient. --- gst/compositor/Makefile.am | 1 - 1 file changed, 1 deletion(-) diff --git a/gst/compositor/Makefile.am b/gst/compositor/Makefile.am index eee77d3d08..5d8e184d47 100644 --- a/gst/compositor/Makefile.am +++ b/gst/compositor/Makefile.am @@ -22,7 +22,6 @@ libgstcompositor_la_LIBADD = \ -lgstvideo-@GST_API_VERSION@ \ $(GST_BASE_LIBS) $(GST_LIBS) $(ORC_LIBS) $(LIBM) libgstcompositor_la_LDFLAGS = $(GST_PLUGIN_LDFLAGS) -libgstcompositor_la_LIBTOOLFLAGS = $(GST_PLUGIN_LIBTOOLFLAGS) # headers we need but don't want installed noinst_HEADERS = \ From cbafb022aad17976d4da44cee13a4872872c8441 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Olivier=20Cr=C3=AAte?= Date: Wed, 6 Jul 2016 17:28:11 -0400 Subject: [PATCH 294/381] tests: Test caps using query Sending an event can accepted event if the caps were rejected because the event could be queued and processed later. Also send a drain query in the caps test to make sure that the event has been processed. https://bugzilla.gnome.org/show_bug.cgi?id=781673 --- tests/check/elements/compositor.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/tests/check/elements/compositor.c b/tests/check/elements/compositor.c index 97211493d9..95c518f04b 100644 --- a/tests/check/elements/compositor.c +++ b/tests/check/elements/compositor.c @@ -385,6 +385,7 @@ GST_START_TEST (test_caps_query_interlaced) GstCaps *caps; GstCaps *caps_mixed, *caps_progressive, *caps_interleaved; GstEvent *caps_event; + GstQuery *drain; caps_interleaved = gst_caps_from_string ("video/x-raw, interlace-mode=interleaved"); @@ -420,6 +421,11 @@ GST_START_TEST (test_caps_query_interlaced) gst_caps_unref (caps); fail_unless (gst_pad_send_event (sinkpad, caps_event)); + /* Send drain query to make sure this is processed */ + drain = gst_query_new_drain (); + gst_pad_query (sinkpad, drain); + gst_query_unref (drain); + /* now recheck the interlace-mode */ gst_object_unref (sinkpad); sinkpad = gst_element_get_request_pad (compositor, "sink_%u"); @@ -565,7 +571,6 @@ run_late_caps_set_test (GstCaps * first_caps, GstCaps * expected_query_caps, GstStateChangeReturn state_res; GstPad *sinkpad_2; GstCaps *caps; - GstEvent *caps_event; GstBus *bus; GstMessage *msg; @@ -596,8 +601,9 @@ run_late_caps_set_test (GstCaps * first_caps, GstCaps * expected_query_caps, sinkpad_2 = gst_element_get_request_pad (compositor, "sink_%u"); caps = gst_pad_query_caps (sinkpad_2, NULL); fail_unless (gst_caps_is_subset (expected_query_caps, caps)); - caps_event = gst_event_new_caps (second_caps); - fail_unless (gst_pad_send_event (sinkpad_2, caps_event) == accept_caps); + gst_caps_unref (caps); + caps = gst_pad_query_caps (sinkpad_2, second_caps); + fail_unless (gst_caps_is_empty (caps) != accept_caps); gst_caps_unref (caps); gst_object_unref (sinkpad_2); From 2a60a9f66fe3754ca1799eba5f08de28744572ec Mon Sep 17 00:00:00 2001 From: George Kiagiadakis Date: Tue, 4 Apr 2017 11:25:43 +0300 Subject: [PATCH 295/381] videoaggregator: delay using new caps from a sink pad until the next buffer in the queue is taken When caps changes while streaming, the new caps was getting processed immediately in videoaggregator, but the next buffer in the queue that corresponds to this new caps was not necessarily being used immediately, which resulted sometimes in using an old buffer with new caps. Of course there used to be a separate buffer_vinfo for mapping the buffer with its own caps, but in compositor the GstVideoConverter was still using wrong info and resulted in invalid reads and corrupt output. This approach here is more safe. We delay using the new caps until we actually select the next buffer in the queue for use. This way we also eliminate the need for buffer_vinfo, since the pad->info is always in sync with the format of the selected buffer. https://bugzilla.gnome.org/show_bug.cgi?id=780682 --- gst-libs/gst/video/gstvideoaggregator.c | 64 ++++++++++++++++++++----- gst/compositor/compositor.c | 21 ++++---- 2 files changed, 62 insertions(+), 23 deletions(-) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index 1616b27244..8310fd2175 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -75,6 +75,8 @@ struct _GstVideoAggregatorPadPrivate GstClockTime start_time; GstClockTime end_time; + + GstVideoInfo pending_vinfo; }; @@ -244,8 +246,7 @@ gst_video_aggregator_pad_prepare_frame (GstVideoAggregatorPad * pad, frame = g_slice_new0 (GstVideoFrame); - if (!gst_video_frame_map (frame, &pad->buffer_vinfo, pad->buffer, - GST_MAP_READ)) { + if (!gst_video_frame_map (frame, &pad->info, pad->buffer, GST_MAP_READ)) { GST_WARNING_OBJECT (vagg, "Could not map input buffer"); return FALSE; } @@ -886,8 +887,20 @@ gst_video_aggregator_pad_sink_setcaps (GstPad * pad, GstObject * parent, } } - vaggpad->info = info; - gst_pad_mark_reconfigure (GST_AGGREGATOR_SRC_PAD (vagg)); + if (!vaggpad->info.finfo || + GST_VIDEO_INFO_FORMAT (&vaggpad->info) == GST_VIDEO_FORMAT_UNKNOWN) { + /* no video info was already set, so this is the first time + * that this pad is getting configured; configure immediately to avoid + * problems with the initial negotiation */ + vaggpad->info = info; + gst_pad_mark_reconfigure (GST_AGGREGATOR_SRC_PAD (vagg)); + } else { + /* this pad already had caps but received new ones; keep the new caps + * pending until we pick the next buffer from the queue, otherwise we + * might use an old buffer with the new caps and crash */ + vaggpad->priv->pending_vinfo = info; + GST_DEBUG_OBJECT (pad, "delaying caps change"); + } ret = TRUE; GST_VIDEO_AGGREGATOR_UNLOCK (vagg); @@ -1098,6 +1111,7 @@ gst_video_aggregator_fill_queues (GstVideoAggregator * vagg, GList *l; gboolean eos = TRUE; gboolean need_more_data = FALSE; + gboolean need_reconfigure = FALSE; /* get a set of buffers into pad->buffer that are within output_start_running_time * and output_end_running_time taking into account finished and unresponsive pads */ @@ -1108,7 +1122,6 @@ gst_video_aggregator_fill_queues (GstVideoAggregator * vagg, GstSegment segment; GstAggregatorPad *bpad; GstBuffer *buf; - GstVideoInfo *vinfo; gboolean is_eos; bpad = GST_AGGREGATOR_PAD (pad); @@ -1131,8 +1144,6 @@ gst_video_aggregator_fill_queues (GstVideoAggregator * vagg, return GST_FLOW_ERROR; } - vinfo = &pad->info; - /* FIXME: Make all this work with negative rates */ end_time = GST_BUFFER_DURATION (buf); @@ -1155,7 +1166,11 @@ gst_video_aggregator_fill_queues (GstVideoAggregator * vagg, GST_DEBUG_OBJECT (pad, "buffer duration is -1, start_time < " "output_start_running_time. Discarding old buffer"); gst_buffer_replace (&pad->buffer, buf); - pad->buffer_vinfo = *vinfo; + if (pad->priv->pending_vinfo.finfo) { + pad->info = pad->priv->pending_vinfo; + need_reconfigure = TRUE; + pad->priv->pending_vinfo.finfo = NULL; + } gst_buffer_unref (buf); gst_aggregator_pad_drop_buffer (bpad); need_more_data = TRUE; @@ -1164,7 +1179,11 @@ gst_video_aggregator_fill_queues (GstVideoAggregator * vagg, gst_buffer_unref (buf); buf = gst_aggregator_pad_steal_buffer (bpad); gst_buffer_replace (&pad->buffer, buf); - pad->buffer_vinfo = *vinfo; + if (pad->priv->pending_vinfo.finfo) { + pad->info = pad->priv->pending_vinfo; + need_reconfigure = TRUE; + pad->priv->pending_vinfo.finfo = NULL; + } /* FIXME: Set start_time and end_time to something here? */ gst_buffer_unref (buf); GST_DEBUG_OBJECT (pad, "buffer duration is -1"); @@ -1225,7 +1244,11 @@ gst_video_aggregator_fill_queues (GstVideoAggregator * vagg, "Taking new buffer with start time %" GST_TIME_FORMAT, GST_TIME_ARGS (start_time)); gst_buffer_replace (&pad->buffer, buf); - pad->buffer_vinfo = *vinfo; + if (pad->priv->pending_vinfo.finfo) { + pad->info = pad->priv->pending_vinfo; + need_reconfigure = TRUE; + pad->priv->pending_vinfo.finfo = NULL; + } pad->priv->start_time = start_time; pad->priv->end_time = end_time; @@ -1239,7 +1262,11 @@ gst_video_aggregator_fill_queues (GstVideoAggregator * vagg, eos = FALSE; } else { gst_buffer_replace (&pad->buffer, buf); - pad->buffer_vinfo = *vinfo; + if (pad->priv->pending_vinfo.finfo) { + pad->info = pad->priv->pending_vinfo; + need_reconfigure = TRUE; + pad->priv->pending_vinfo.finfo = NULL; + } pad->priv->start_time = start_time; pad->priv->end_time = end_time; GST_DEBUG_OBJECT (pad, @@ -1278,6 +1305,9 @@ gst_video_aggregator_fill_queues (GstVideoAggregator * vagg, } GST_OBJECT_UNLOCK (vagg); + if (need_reconfigure) + gst_pad_mark_reconfigure (GST_AGGREGATOR_SRC_PAD (vagg)); + if (need_more_data) return GST_FLOW_NEEDS_DATA; if (eos) @@ -1515,6 +1545,7 @@ gst_video_aggregator_aggregate (GstAggregator * agg, gboolean timeout) GST_VIDEO_AGGREGATOR_LOCK (vagg); +restart: flow_ret = gst_video_aggregator_check_reconfigure (vagg, timeout); if (flow_ret != GST_FLOW_OK) { if (flow_ret == GST_FLOW_NEEDS_DATA) @@ -1572,6 +1603,17 @@ gst_video_aggregator_aggregate (GstAggregator * agg, gboolean timeout) goto unlock_and_return; } + /* It is possible that gst_video_aggregator_fill_queues() marked the pad + * for reconfiguration. In this case we have to reconfigure before continuing + * because we have picked a new buffer with different caps than before from + * one one of the sink pads and continuing here may lead to a crash. + * https://bugzilla.gnome.org/show_bug.cgi?id=780682 + */ + if (gst_pad_needs_reconfigure (GST_AGGREGATOR_SRC_PAD (vagg))) { + GST_DEBUG_OBJECT (vagg, "Need reconfigure"); + goto restart; + } + GST_DEBUG_OBJECT (vagg, "Producing buffer for %" GST_TIME_FORMAT " to %" GST_TIME_FORMAT ", running time start %" GST_TIME_FORMAT ", running time end %" diff --git a/gst/compositor/compositor.c b/gst/compositor/compositor.c index fab75c6394..de3bf22d65 100644 --- a/gst/compositor/compositor.c +++ b/gst/compositor/compositor.c @@ -381,7 +381,7 @@ gst_compositor_pad_prepare_frame (GstVideoAggregatorPad * pad, /* There's three types of width/height here: * 1. GST_VIDEO_FRAME_WIDTH/HEIGHT: - * The frame width/height (same as pad->buffer_vinfo.height/width; + * The frame width/height (same as pad->info.height/width; * see gst_video_frame_map()) * 2. cpad->width/height: * The optional pad property for scaling the frame (if zero, the video is @@ -409,21 +409,20 @@ gst_compositor_pad_prepare_frame (GstVideoAggregatorPad * pad, gst_video_converter_free (cpad->convert); cpad->convert = NULL; - colorimetry = - gst_video_colorimetry_to_string (&pad->buffer_vinfo.colorimetry); - chroma = gst_video_chroma_to_string (pad->buffer_vinfo.chroma_site); + colorimetry = gst_video_colorimetry_to_string (&pad->info.colorimetry); + chroma = gst_video_chroma_to_string (pad->info.chroma_site); wanted_colorimetry = gst_video_colorimetry_to_string (&cpad->conversion_info.colorimetry); wanted_chroma = gst_video_chroma_to_string (cpad->conversion_info.chroma_site); - if (GST_VIDEO_INFO_FORMAT (&pad->buffer_vinfo) != + if (GST_VIDEO_INFO_FORMAT (&pad->info) != GST_VIDEO_INFO_FORMAT (&cpad->conversion_info) || g_strcmp0 (colorimetry, wanted_colorimetry) || g_strcmp0 (chroma, wanted_chroma) - || width != GST_VIDEO_INFO_WIDTH (&pad->buffer_vinfo) - || height != GST_VIDEO_INFO_HEIGHT (&pad->buffer_vinfo)) { + || width != GST_VIDEO_INFO_WIDTH (&pad->info) + || height != GST_VIDEO_INFO_HEIGHT (&pad->info)) { GstVideoInfo tmp_info; gst_video_info_set_format (&tmp_info, cpad->conversion_info.finfo->format, @@ -438,11 +437,10 @@ gst_compositor_pad_prepare_frame (GstVideoAggregatorPad * pad, tmp_info.interlace_mode = cpad->conversion_info.interlace_mode; GST_DEBUG_OBJECT (pad, "This pad will be converted from %d to %d", - GST_VIDEO_INFO_FORMAT (&pad->buffer_vinfo), + GST_VIDEO_INFO_FORMAT (&pad->info), GST_VIDEO_INFO_FORMAT (&tmp_info)); - cpad->convert = - gst_video_converter_new (&pad->buffer_vinfo, &tmp_info, NULL); + cpad->convert = gst_video_converter_new (&pad->info, &tmp_info, NULL); cpad->conversion_info = tmp_info; if (!cpad->convert) { @@ -522,8 +520,7 @@ gst_compositor_pad_prepare_frame (GstVideoAggregatorPad * pad, frame = g_slice_new0 (GstVideoFrame); - if (!gst_video_frame_map (frame, &pad->buffer_vinfo, pad->buffer, - GST_MAP_READ)) { + if (!gst_video_frame_map (frame, &pad->info, pad->buffer, GST_MAP_READ)) { GST_WARNING_OBJECT (vagg, "Could not map input buffer"); return FALSE; } From 719498601f0156163a7798a5ad4b88999234645e Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Sat, 20 May 2017 14:24:57 +0200 Subject: [PATCH 296/381] aggregator: add simple support for caps handling Modelled off the videoaggregator caps handling as that seems the most mature aggregtor-using implementation that has caps handling there is. https://bugzilla.gnome.org/show_bug.cgi?id=776931 --- ext/gl/gstglbasemixer.c | 15 +- ext/gl/gstglmixer.c | 32 +-- ext/gl/gstglstereomix.c | 18 +- ext/gl/gstglvideomixer.c | 18 +- gst-libs/gst/video/gstvideoaggregator.c | 360 +++++++++--------------- gst-libs/gst/video/gstvideoaggregator.h | 8 - gst/compositor/compositor.c | 17 +- 7 files changed, 166 insertions(+), 302 deletions(-) diff --git a/ext/gl/gstglbasemixer.c b/ext/gl/gstglbasemixer.c index 6e6e9493e0..317d339ed4 100644 --- a/ext/gl/gstglbasemixer.c +++ b/ext/gl/gstglbasemixer.c @@ -108,11 +108,14 @@ gst_gl_base_mixer_pad_set_property (GObject * object, guint prop_id, } static gboolean -_negotiated_caps (GstVideoAggregator * vagg, GstCaps * caps) +_negotiated_caps (GstAggregator * agg, GstCaps * caps) { - GstGLBaseMixer *mix = GST_GL_BASE_MIXER (vagg); + GstGLBaseMixer *mix = GST_GL_BASE_MIXER (agg); - return gst_gl_base_mixer_do_bufferpool (mix, caps); + if (!gst_gl_base_mixer_do_bufferpool (mix, caps)) + return FALSE; + + return GST_AGGREGATOR_CLASS (parent_class)->negotiated_src_caps (agg, caps); } static gboolean @@ -324,9 +327,6 @@ gst_gl_base_mixer_class_init (GstGLBaseMixerClass * klass) { GObjectClass *gobject_class; GstElementClass *element_class; - - GstVideoAggregatorClass *videoaggregator_class = - (GstVideoAggregatorClass *) klass; GstAggregatorClass *agg_class = (GstAggregatorClass *) klass; GST_DEBUG_CATEGORY_INIT (GST_CAT_DEFAULT, "glmixer", 0, "opengl mixer"); @@ -350,8 +350,7 @@ gst_gl_base_mixer_class_init (GstGLBaseMixerClass * klass) agg_class->src_activate = gst_gl_base_mixer_src_activate_mode; agg_class->stop = gst_gl_base_mixer_stop; agg_class->start = gst_gl_base_mixer_start; - - videoaggregator_class->negotiated_caps = _negotiated_caps; + agg_class->negotiated_src_caps = _negotiated_caps; klass->propose_allocation = _default_propose_allocation; diff --git a/ext/gl/gstglmixer.c b/ext/gl/gstglmixer.c index b8f4da0267..d2ce49da8a 100644 --- a/ext/gl/gstglmixer.c +++ b/ext/gl/gstglmixer.c @@ -96,16 +96,16 @@ gst_gl_mixer_pad_set_property (GObject * object, guint prop_id, } static gboolean -_negotiated_caps (GstVideoAggregator * vagg, GstCaps * caps) +_negotiated_caps (GstAggregator * agg, GstCaps * caps) { - GstGLMixer *mix = GST_GL_MIXER (vagg); + GstGLMixer *mix = GST_GL_MIXER (agg); gboolean ret; mix->priv->negotiated = TRUE; gst_caps_replace (&mix->out_caps, caps); - ret = GST_VIDEO_AGGREGATOR_CLASS (parent_class)->negotiated_caps (vagg, caps); + ret = GST_AGGREGATOR_CLASS (parent_class)->negotiated_src_caps (agg, caps); return ret; } @@ -215,29 +215,6 @@ gst_gl_mixer_pad_sink_acceptcaps (GstPad * pad, GstGLMixer * mix, return ret; } -/* copies the given caps */ -static GstCaps * -_update_caps (GstVideoAggregator * vagg, GstCaps * caps, GstCaps * filter) -{ - GstCaps *tmp; - guint i, n; - - if (filter) { - tmp = gst_caps_intersect (caps, filter); - tmp = gst_caps_make_writable (tmp); - } else { - tmp = gst_caps_copy (caps); - } - - n = gst_caps_get_size (tmp); - for (i = 0; i < n; i++) { - gst_caps_set_features (tmp, i, - gst_caps_features_from_string (GST_CAPS_FEATURE_MEMORY_GL_MEMORY)); - } - - return tmp; -} - static GstCaps * gst_gl_mixer_pad_sink_getcaps (GstPad * pad, GstGLMixer * mix, GstCaps * filter) { @@ -391,11 +368,10 @@ gst_gl_mixer_class_init (GstGLMixerClass * klass) agg_class->src_query = gst_gl_mixer_src_query; agg_class->stop = gst_gl_mixer_stop; agg_class->start = gst_gl_mixer_start; + agg_class->negotiated_src_caps = _negotiated_caps; videoaggregator_class->aggregate_frames = gst_gl_mixer_aggregate_frames; videoaggregator_class->get_output_buffer = gst_gl_mixer_get_output_buffer; - videoaggregator_class->negotiated_caps = _negotiated_caps; - videoaggregator_class->update_caps = _update_caps; videoaggregator_class->find_best_format = _find_best_format; mix_class->propose_allocation = gst_gl_mixer_propose_allocation; diff --git a/ext/gl/gstglstereomix.c b/ext/gl/gstglstereomix.c index 78a2934fad..9e42b23f67 100644 --- a/ext/gl/gstglstereomix.c +++ b/ext/gl/gstglstereomix.c @@ -81,10 +81,8 @@ gst_gl_stereo_mix_pad_init (GstGLStereoMixPad * pad) #define gst_gl_stereo_mix_parent_class parent_class G_DEFINE_TYPE (GstGLStereoMix, gst_gl_stereo_mix, GST_TYPE_GL_MIXER); -static GstCaps *_update_caps (GstVideoAggregator * vagg, GstCaps * caps, - GstCaps * filter); -static gboolean _negotiated_caps (GstVideoAggregator * videoaggregator, - GstCaps * caps); +static GstCaps *_update_caps (GstVideoAggregator * vagg, GstCaps * caps); +static gboolean _negotiated_caps (GstAggregator * aggregator, GstCaps * caps); gboolean gst_gl_stereo_mix_make_output (GstGLStereoMix * mix); static gboolean gst_gl_stereo_mix_process_frames (GstGLStereoMix * mixer); @@ -188,10 +186,10 @@ gst_gl_stereo_mix_class_init (GstGLStereoMixClass * klass) agg_class->stop = gst_gl_stereo_mix_stop; agg_class->start = gst_gl_stereo_mix_start; agg_class->src_query = gst_gl_stereo_mix_src_query; + agg_class->negotiated_src_caps = _negotiated_caps; videoaggregator_class->aggregate_frames = gst_gl_stereo_mix_aggregate_frames; videoaggregator_class->update_caps = _update_caps; - videoaggregator_class->negotiated_caps = _negotiated_caps; videoaggregator_class->get_output_buffer = gst_gl_stereo_mix_get_output_buffer; @@ -470,7 +468,7 @@ get_converted_caps (GstGLStereoMix * mix, GstCaps * caps) /* Return the possible output caps based on inputs and downstream prefs */ static GstCaps * -_update_caps (GstVideoAggregator * vagg, GstCaps * caps, GstCaps * filter) +_update_caps (GstVideoAggregator * vagg, GstCaps * caps) { GstGLStereoMix *mix = GST_GL_STEREO_MIX (vagg); GList *l; @@ -563,16 +561,16 @@ _update_caps (GstVideoAggregator * vagg, GstCaps * caps, GstCaps * filter) /* Called after videoaggregator fixates our caps */ static gboolean -_negotiated_caps (GstVideoAggregator * vagg, GstCaps * caps) +_negotiated_caps (GstAggregator * agg, GstCaps * caps) { + GstVideoAggregator *vagg = GST_VIDEO_AGGREGATOR (agg); GstGLStereoMix *mix = GST_GL_STEREO_MIX (vagg); GstCaps *in_caps; GST_LOG_OBJECT (mix, "Configured output caps %" GST_PTR_FORMAT, caps); - if (GST_VIDEO_AGGREGATOR_CLASS (parent_class)->negotiated_caps) - if (!GST_VIDEO_AGGREGATOR_CLASS (parent_class)->negotiated_caps (vagg, - caps)) + if (GST_AGGREGATOR_CLASS (parent_class)->negotiated_src_caps) + if (!GST_AGGREGATOR_CLASS (parent_class)->negotiated_src_caps (agg, caps)) return FALSE; /* Update the glview_convert output */ diff --git a/ext/gl/gstglvideomixer.c b/ext/gl/gstglvideomixer.c index 06bdf3e102..10251e6792 100644 --- a/ext/gl/gstglvideomixer.c +++ b/ext/gl/gstglvideomixer.c @@ -453,9 +453,8 @@ static void gst_gl_video_mixer_set_property (GObject * object, guint prop_id, static void gst_gl_video_mixer_get_property (GObject * object, guint prop_id, GValue * value, GParamSpec * pspec); -static GstCaps *_update_caps (GstVideoAggregator * vagg, GstCaps * caps, - GstCaps * filter); -static GstCaps *_fixate_caps (GstVideoAggregator * vagg, GstCaps * caps); +static GstCaps *_update_caps (GstVideoAggregator * vagg, GstCaps * caps); +static GstCaps *_fixate_caps (GstAggregator * agg, GstCaps * caps); static gboolean gst_gl_video_mixer_propose_allocation (GstGLBaseMixer * base_mix, GstGLBaseMixerPad * base_pad, GstQuery * decide_query, GstQuery * query); @@ -874,9 +873,9 @@ gst_gl_video_mixer_class_init (GstGLVideoMixerClass * klass) gst_gl_video_mixer_process_textures; vagg_class->update_caps = _update_caps; - vagg_class->fixate_caps = _fixate_caps; agg_class->sinkpads_type = GST_TYPE_GL_VIDEO_MIXER_PAD; + agg_class->fixate_src_caps = _fixate_caps; mix_class->propose_allocation = gst_gl_video_mixer_propose_allocation; @@ -986,7 +985,7 @@ _mixer_pad_get_output_size (GstGLVideoMixer * mix, } static GstCaps * -_update_caps (GstVideoAggregator * vagg, GstCaps * caps, GstCaps * filter) +_update_caps (GstVideoAggregator * vagg, GstCaps * caps) { GstCaps *ret; GList *l; @@ -1014,18 +1013,15 @@ _update_caps (GstVideoAggregator * vagg, GstCaps * caps, GstCaps * filter) GST_OBJECT_UNLOCK (vagg); - if (filter) { - ret = gst_caps_intersect (caps, filter); - } else { - ret = gst_caps_ref (caps); - } + ret = gst_caps_ref (caps); return ret; } static GstCaps * -_fixate_caps (GstVideoAggregator * vagg, GstCaps * caps) +_fixate_caps (GstAggregator * agg, GstCaps * caps) { + GstVideoAggregator *vagg = GST_VIDEO_AGGREGATOR (agg); GstGLVideoMixer *mix = GST_GL_VIDEO_MIXER (vagg); gint best_width = 0, best_height = 0; gint best_fps_n = 0, best_fps_d = 0; diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index 8310fd2175..7cdce00fa6 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -535,60 +535,11 @@ gst_video_aggregator_find_best_format (GstVideoAggregator * vagg, g_hash_table_unref (formats_table); } -/* WITH GST_VIDEO_AGGREGATOR_LOCK TAKEN */ -static gboolean -gst_video_aggregator_src_setcaps (GstVideoAggregator * vagg, GstCaps * caps) -{ - GstAggregator *agg = GST_AGGREGATOR (vagg); - gboolean ret = FALSE; - GstVideoInfo info; - - GstPad *pad = GST_AGGREGATOR (vagg)->srcpad; - - GST_INFO_OBJECT (pad, "set src caps: %" GST_PTR_FORMAT, caps); - - if (!gst_video_info_from_caps (&info, caps)) - goto done; - - ret = TRUE; - - if (GST_VIDEO_INFO_FPS_N (&vagg->info) != GST_VIDEO_INFO_FPS_N (&info) || - GST_VIDEO_INFO_FPS_D (&vagg->info) != GST_VIDEO_INFO_FPS_D (&info)) { - if (agg->segment.position != -1) { - vagg->priv->nframes = 0; - /* The timestamp offset will be updated based on the - * segment position the next time we aggregate */ - GST_DEBUG_OBJECT (vagg, - "Resetting frame counter because of framerate change"); - } - gst_video_aggregator_reset_qos (vagg); - } - - vagg->info = info; - - if (vagg->priv->current_caps == NULL || - gst_caps_is_equal (caps, vagg->priv->current_caps) == FALSE) { - GstClockTime latency; - - gst_caps_replace (&vagg->priv->current_caps, caps); - GST_VIDEO_AGGREGATOR_UNLOCK (vagg); - - gst_aggregator_set_src_caps (agg, caps); - latency = gst_util_uint64_scale (GST_SECOND, - GST_VIDEO_INFO_FPS_D (&info), GST_VIDEO_INFO_FPS_N (&info)); - gst_aggregator_set_latency (agg, latency, latency); - - GST_VIDEO_AGGREGATOR_LOCK (vagg); - } - -done: - return ret; -} - static GstCaps * -gst_video_aggregator_default_fixate_caps (GstVideoAggregator * vagg, +gst_video_aggregator_default_fixate_src_caps (GstAggregator * agg, GstCaps * caps) { + GstVideoAggregator *vagg = GST_VIDEO_AGGREGATOR (agg); gint best_width = -1, best_height = -1; gint best_fps_n = -1, best_fps_d = -1; gdouble best_fps = -1.; @@ -634,6 +585,7 @@ gst_video_aggregator_default_fixate_caps (GstVideoAggregator * vagg, best_fps = 25.0; } + caps = gst_caps_make_writable (caps); s = gst_caps_get_structure (caps, 0); gst_structure_fixate_field_nearest_int (s, "width", best_width); gst_structure_fixate_field_nearest_int (s, "height", best_height); @@ -648,7 +600,7 @@ gst_video_aggregator_default_fixate_caps (GstVideoAggregator * vagg, static GstCaps * gst_video_aggregator_default_update_caps (GstVideoAggregator * vagg, - GstCaps * caps, GstCaps * filter) + GstCaps * caps) { GstVideoAggregatorClass *vagg_klass = GST_VIDEO_AGGREGATOR_GET_CLASS (vagg); GstCaps *ret, *best_format_caps; @@ -682,38 +634,59 @@ gst_video_aggregator_default_update_caps (GstVideoAggregator * vagg, gst_video_chroma_to_string (best_info.chroma_site), NULL); ret = gst_caps_merge (best_format_caps, gst_caps_ref (caps)); - if (filter) { - GstCaps *tmp; - tmp = gst_caps_intersect (ret, filter); - gst_caps_unref (ret); - ret = tmp; - } - return ret; } -/* WITH GST_VIDEO_AGGREGATOR_LOCK TAKEN */ -static gboolean -gst_video_aggregator_update_src_caps (GstVideoAggregator * vagg) +static GstFlowReturn +gst_video_aggregator_default_update_src_caps (GstAggregator * agg, + GstCaps * caps, GstCaps ** ret) { - GstVideoAggregatorClass *vagg_klass = GST_VIDEO_AGGREGATOR_GET_CLASS (vagg); - GstVideoAggregatorPadClass *vaggpad_klass = g_type_class_peek - (GST_AGGREGATOR_GET_CLASS (vagg)->sinkpads_type); - GstAggregator *agg = GST_AGGREGATOR (vagg); - gboolean ret = TRUE, at_least_one_pad_configured = FALSE; - gboolean at_least_one_alpha = FALSE; - GstCaps *downstream_caps; + GstVideoAggregatorClass *vagg_klass = GST_VIDEO_AGGREGATOR_GET_CLASS (agg); + GstVideoAggregator *vagg = GST_VIDEO_AGGREGATOR (agg); + gboolean at_least_one_pad_configured = FALSE; GList *l; - downstream_caps = gst_pad_get_allowed_caps (agg->srcpad); + GST_OBJECT_LOCK (vagg); + for (l = GST_ELEMENT (vagg)->sinkpads; l; l = l->next) { + GstVideoAggregatorPad *mpad = l->data; - if (!downstream_caps || gst_caps_is_empty (downstream_caps)) { - GST_INFO_OBJECT (vagg, "No downstream caps found %" - GST_PTR_FORMAT, downstream_caps); - if (downstream_caps) - gst_caps_unref (downstream_caps); - return FALSE; + if (GST_VIDEO_INFO_WIDTH (&mpad->info) == 0 + || GST_VIDEO_INFO_HEIGHT (&mpad->info) == 0) + continue; + + at_least_one_pad_configured = TRUE; } + GST_OBJECT_UNLOCK (vagg); + + if (!at_least_one_pad_configured) { + /* We couldn't decide the output video info because the sinkpads don't have + * all the caps yet, so we mark the pad as needing a reconfigure. This + * allows aggregate() to skip ahead a bit and try again later. */ + GST_DEBUG_OBJECT (vagg, "Couldn't decide output video info"); + gst_pad_mark_reconfigure (agg->srcpad); + return GST_AGGREGATOR_FLOW_NEED_DATA; + } + + g_assert (vagg_klass->update_caps); + + *ret = vagg_klass->update_caps (vagg, caps); + + return GST_FLOW_OK; +} + +static gboolean +gst_video_aggregator_default_negotiated_src_caps (GstAggregator * agg, + GstCaps * caps) +{ + GstVideoAggregatorPadClass *vaggpad_klass = g_type_class_peek + (GST_AGGREGATOR_GET_CLASS (agg)->sinkpads_type); + GstVideoAggregator *vagg = GST_VIDEO_AGGREGATOR (agg); + gboolean at_least_one_alpha = FALSE; + const GstVideoFormatInfo *finfo; + GstVideoInfo info; + GList *l; + + GST_INFO_OBJECT (agg->srcpad, "set src caps: %" GST_PTR_FORMAT, caps); GST_OBJECT_LOCK (vagg); for (l = GST_ELEMENT (vagg)->sinkpads; l; l = l->next) { @@ -725,101 +698,59 @@ gst_video_aggregator_update_src_caps (GstVideoAggregator * vagg) if (mpad->info.finfo->flags & GST_VIDEO_FORMAT_FLAG_ALPHA) at_least_one_alpha = TRUE; - - at_least_one_pad_configured = TRUE; } GST_OBJECT_UNLOCK (vagg); - if (at_least_one_pad_configured) { - GstCaps *caps, *peercaps; + if (!gst_video_info_from_caps (&info, caps)) + return FALSE; - peercaps = gst_pad_peer_query_caps (agg->srcpad, NULL); - - g_assert (vagg_klass->update_caps); - GST_DEBUG_OBJECT (vagg, "updating caps from %" GST_PTR_FORMAT, - downstream_caps); - GST_DEBUG_OBJECT (vagg, " with filter %" GST_PTR_FORMAT, peercaps); - if (!(caps = vagg_klass->update_caps (vagg, downstream_caps, peercaps)) || - gst_caps_is_empty (caps)) { - GST_WARNING_OBJECT (vagg, "Subclass failed to update provided caps"); - gst_caps_unref (downstream_caps); - if (peercaps) - gst_caps_unref (peercaps); - ret = FALSE; - goto done; + if (GST_VIDEO_INFO_FPS_N (&vagg->info) != GST_VIDEO_INFO_FPS_N (&info) || + GST_VIDEO_INFO_FPS_D (&vagg->info) != GST_VIDEO_INFO_FPS_D (&info)) { + if (agg->segment.position != -1) { + vagg->priv->nframes = 0; + /* The timestamp offset will be updated based on the + * segment position the next time we aggregate */ + GST_DEBUG_OBJECT (vagg, + "Resetting frame counter because of framerate change"); } - GST_DEBUG_OBJECT (vagg, " to %" GST_PTR_FORMAT, caps); - - gst_caps_unref (downstream_caps); - if (peercaps) - gst_caps_unref (peercaps); - - if (!gst_caps_is_fixed (caps)) { - g_assert (vagg_klass->fixate_caps); - - caps = gst_caps_make_writable (caps); - GST_DEBUG_OBJECT (vagg, "fixate caps from %" GST_PTR_FORMAT, caps); - if (!(caps = vagg_klass->fixate_caps (vagg, caps))) { - GST_WARNING_OBJECT (vagg, "Subclass failed to fixate provided caps"); - ret = FALSE; - goto done; - } - GST_DEBUG_OBJECT (vagg, " to %" GST_PTR_FORMAT, caps); - } - - { - const GstVideoFormatInfo *finfo; - const gchar *v_format_str; - GstVideoFormat v_format; - GstStructure *s; - - s = gst_caps_get_structure (caps, 0); - v_format_str = gst_structure_get_string (s, "format"); - g_return_val_if_fail (v_format_str != NULL, FALSE); - v_format = gst_video_format_from_string (v_format_str); - g_return_val_if_fail (v_format != GST_VIDEO_FORMAT_UNKNOWN, FALSE); - finfo = gst_video_format_get_info (v_format); - g_return_val_if_fail (finfo != NULL, FALSE); - - if (at_least_one_alpha && !(finfo->flags & GST_VIDEO_FORMAT_FLAG_ALPHA)) { - GST_ELEMENT_ERROR (vagg, CORE, NEGOTIATION, - ("At least one of the input pads contains alpha, but configured caps don't support alpha."), - ("Either convert your inputs to not contain alpha or add a videoconvert after the aggregator")); - ret = FALSE; - goto done; - } - } - - gst_video_info_from_caps (&vagg->info, caps); - - if (vaggpad_klass->set_info) { - /* Then browse the sinks once more, setting or unsetting conversion if needed */ - for (l = GST_ELEMENT (vagg)->sinkpads; l; l = l->next) { - GstVideoAggregatorPad *pad = GST_VIDEO_AGGREGATOR_PAD (l->data); - - if (!vaggpad_klass->set_info (pad, vagg, &pad->info, &vagg->info)) { - return FALSE; - } - } - } - - if (gst_video_aggregator_src_setcaps (vagg, caps)) { - if (vagg_klass->negotiated_caps) - ret = - GST_VIDEO_AGGREGATOR_GET_CLASS (vagg)->negotiated_caps (vagg, caps); - } - gst_caps_unref (caps); - } else { - /* We couldn't decide the output video info because the sinkpads don't have - * all the caps yet, so we mark the pad as needing a reconfigure. This - * allows aggregate() to skip ahead a bit and try again later. */ - GST_DEBUG_OBJECT (vagg, "Couldn't decide output video info"); - gst_pad_mark_reconfigure (agg->srcpad); - ret = FALSE; + gst_video_aggregator_reset_qos (vagg); } -done: - return ret; + vagg->info = info; + + finfo = vagg->info.finfo; + + if (at_least_one_alpha && !(finfo->flags & GST_VIDEO_FORMAT_FLAG_ALPHA)) { + GST_ELEMENT_ERROR (vagg, CORE, NEGOTIATION, + ("At least one of the input pads contains alpha, but configured caps don't support alpha."), + ("Either convert your inputs to not contain alpha or add a videoconvert after the aggregator")); + return FALSE; + } + + if (vaggpad_klass->set_info) { + /* Then browse the sinks once more, setting or unsetting conversion if needed */ + for (l = GST_ELEMENT (vagg)->sinkpads; l; l = l->next) { + GstVideoAggregatorPad *pad = GST_VIDEO_AGGREGATOR_PAD (l->data); + + if (!vaggpad_klass->set_info (pad, vagg, &pad->info, &vagg->info)) { + return FALSE; + } + } + } + + if (vagg->priv->current_caps == NULL || + gst_caps_is_equal (caps, vagg->priv->current_caps) == FALSE) { + GstClockTime latency; + + gst_caps_replace (&vagg->priv->current_caps, caps); + + gst_aggregator_set_src_caps (agg, caps); + latency = gst_util_uint64_scale (GST_SECOND, + GST_VIDEO_INFO_FPS_D (&vagg->info), GST_VIDEO_INFO_FPS_N (&vagg->info)); + gst_aggregator_set_latency (agg, latency, latency); + } + + return TRUE; } static gboolean @@ -1101,7 +1032,6 @@ gst_video_aggregator_reset (GstVideoAggregator * vagg) GST_OBJECT_UNLOCK (vagg); } -#define GST_FLOW_NEEDS_DATA GST_FLOW_CUSTOM_ERROR static gint gst_video_aggregator_fill_queues (GstVideoAggregator * vagg, GstClockTime output_start_running_time, @@ -1309,7 +1239,7 @@ gst_video_aggregator_fill_queues (GstVideoAggregator * vagg, gst_pad_mark_reconfigure (GST_AGGREGATOR_SRC_PAD (vagg)); if (need_more_data) - return GST_FLOW_NEEDS_DATA; + return GST_AGGREGATOR_FLOW_NEED_DATA; if (eos) return GST_FLOW_EOS; @@ -1471,66 +1401,36 @@ gst_video_aggregator_get_next_time (GstAggregator * agg) return next_time; } -static GstFlowReturn -gst_video_aggregator_check_reconfigure (GstVideoAggregator * vagg, - gboolean timeout) +static void +gst_video_aggregator_advance_on_timeout (GstVideoAggregator * vagg) { - GstAggregator *agg = (GstAggregator *) vagg; + GstAggregator *agg = GST_AGGREGATOR (vagg); + guint64 frame_duration; + gint fps_d, fps_n; - if (GST_VIDEO_INFO_FORMAT (&vagg->info) == GST_VIDEO_FORMAT_UNKNOWN - || gst_pad_check_reconfigure (GST_AGGREGATOR_SRC_PAD (vagg))) { - gboolean ret; - - restart: - ret = gst_video_aggregator_update_src_caps (vagg); - if (!ret) { - gst_pad_mark_reconfigure (GST_AGGREGATOR_SRC_PAD (vagg)); - if (timeout) { - guint64 frame_duration; - gint fps_d, fps_n; - - GST_DEBUG_OBJECT (vagg, - "Got timeout before receiving any caps, don't output anything"); - - if (agg->segment.position == -1) { - if (agg->segment.rate > 0.0) - agg->segment.position = agg->segment.start; - else - agg->segment.position = agg->segment.stop; - } - - /* Advance position */ - fps_d = GST_VIDEO_INFO_FPS_D (&vagg->info) ? - GST_VIDEO_INFO_FPS_D (&vagg->info) : 1; - fps_n = GST_VIDEO_INFO_FPS_N (&vagg->info) ? - GST_VIDEO_INFO_FPS_N (&vagg->info) : 25; - /* Default to 25/1 if no "best fps" is known */ - frame_duration = gst_util_uint64_scale (GST_SECOND, fps_d, fps_n); - if (agg->segment.rate > 0.0) - agg->segment.position += frame_duration; - else if (agg->segment.position > frame_duration) - agg->segment.position -= frame_duration; - else - agg->segment.position = 0; - vagg->priv->nframes++; - return GST_FLOW_NEEDS_DATA; - } else { - if (GST_PAD_IS_FLUSHING (GST_AGGREGATOR_SRC_PAD (vagg))) - return GST_FLOW_FLUSHING; - else - return GST_FLOW_NOT_NEGOTIATED; - } - } else { - /* It is possible that during gst_video_aggregator_update_src_caps() - * we got a caps change on one of the sink pads, in which case we need - * to redo the negotiation - * - https://bugzilla.gnome.org/show_bug.cgi?id=755782 */ - if (gst_pad_check_reconfigure (GST_AGGREGATOR_SRC_PAD (vagg))) - goto restart; - } + GST_OBJECT_LOCK (agg); + if (agg->segment.position == -1) { + if (agg->segment.rate > 0.0) + agg->segment.position = agg->segment.start; + else + agg->segment.position = agg->segment.stop; } - return GST_FLOW_OK; + /* Advance position */ + fps_d = GST_VIDEO_INFO_FPS_D (&vagg->info) ? + GST_VIDEO_INFO_FPS_D (&vagg->info) : 1; + fps_n = GST_VIDEO_INFO_FPS_N (&vagg->info) ? + GST_VIDEO_INFO_FPS_N (&vagg->info) : 25; + /* Default to 25/1 if no "best fps" is known */ + frame_duration = gst_util_uint64_scale (GST_SECOND, fps_d, fps_n); + if (agg->segment.rate > 0.0) + agg->segment.position += frame_duration; + else if (agg->segment.position > frame_duration) + agg->segment.position -= frame_duration; + else + agg->segment.position = 0; + vagg->priv->nframes++; + GST_OBJECT_UNLOCK (agg); } static GstFlowReturn @@ -1546,10 +1446,10 @@ gst_video_aggregator_aggregate (GstAggregator * agg, gboolean timeout) GST_VIDEO_AGGREGATOR_LOCK (vagg); restart: - flow_ret = gst_video_aggregator_check_reconfigure (vagg, timeout); - if (flow_ret != GST_FLOW_OK) { - if (flow_ret == GST_FLOW_NEEDS_DATA) - flow_ret = GST_FLOW_OK; + if (GST_VIDEO_INFO_FORMAT (&vagg->info) == GST_VIDEO_FORMAT_UNKNOWN) { + if (timeout) + gst_video_aggregator_advance_on_timeout (vagg); + flow_ret = GST_AGGREGATOR_FLOW_NEED_DATA; goto unlock_and_return; } @@ -1591,9 +1491,8 @@ restart: output_end_running_time); } - if (flow_ret == GST_FLOW_NEEDS_DATA && !timeout) { + if (flow_ret == GST_AGGREGATOR_FLOW_NEED_DATA && !timeout) { GST_DEBUG_OBJECT (vagg, "Need more data for decisions"); - flow_ret = GST_FLOW_OK; goto unlock_and_return; } else if (flow_ret == GST_FLOW_EOS) { GST_DEBUG_OBJECT (vagg, "All sinkpads are EOS -- forwarding"); @@ -2187,11 +2086,14 @@ gst_video_aggregator_class_init (GstVideoAggregatorClass * klass) agg_class->src_event = gst_video_aggregator_src_event; agg_class->src_query = gst_video_aggregator_src_query; agg_class->get_next_time = gst_video_aggregator_get_next_time; + agg_class->update_src_caps = gst_video_aggregator_default_update_src_caps; + agg_class->fixate_src_caps = gst_video_aggregator_default_fixate_src_caps; + agg_class->negotiated_src_caps = + gst_video_aggregator_default_negotiated_src_caps; klass->find_best_format = gst_video_aggregator_find_best_format; klass->get_output_buffer = gst_video_aggregator_get_output_buffer; klass->update_caps = gst_video_aggregator_default_update_caps; - klass->fixate_caps = gst_video_aggregator_default_fixate_caps; /* Register the pad class */ g_type_class_ref (GST_TYPE_VIDEO_AGGREGATOR_PAD); diff --git a/gst-libs/gst/video/gstvideoaggregator.h b/gst-libs/gst/video/gstvideoaggregator.h index 6204bdd037..ced6124aec 100644 --- a/gst-libs/gst/video/gstvideoaggregator.h +++ b/gst-libs/gst/video/gstvideoaggregator.h @@ -73,9 +73,6 @@ struct _GstVideoAggregator * @update_caps: Optional. * Lets subclasses update the #GstCaps representing * the src pad caps before usage. Return %NULL to indicate failure. - * @fixate_caps: Fixate and return the src pad caps provided. The function takes - * ownership of @caps and returns a fixated version of - * @caps. @caps is not guaranteed to be writable. * @aggregate_frames: Lets subclasses aggregate frames that are ready. Subclasses * should iterate the GstElement.sinkpads and use the already * mapped #GstVideoFrame from GstVideoAggregatorPad.aggregated_frame @@ -97,16 +94,11 @@ struct _GstVideoAggregatorClass /*< public >*/ GstCaps * (*update_caps) (GstVideoAggregator * videoaggregator, - GstCaps * caps, - GstCaps * filter_caps); - GstCaps * (*fixate_caps) (GstVideoAggregator * videoaggregator, GstCaps * caps); GstFlowReturn (*aggregate_frames) (GstVideoAggregator * videoaggregator, GstBuffer * outbuffer); GstFlowReturn (*get_output_buffer) (GstVideoAggregator * videoaggregator, GstBuffer ** outbuffer); - gboolean (*negotiated_caps) (GstVideoAggregator * videoaggregator, - GstCaps * caps); void (*find_best_format) (GstVideoAggregator * vagg, GstCaps * downstream_caps, GstVideoInfo * best_info, diff --git a/gst/compositor/compositor.c b/gst/compositor/compositor.c index de3bf22d65..23dc20cda1 100644 --- a/gst/compositor/compositor.c +++ b/gst/compositor/compositor.c @@ -870,8 +870,9 @@ set_functions (GstCompositor * self, GstVideoInfo * info) } static GstCaps * -_fixate_caps (GstVideoAggregator * vagg, GstCaps * caps) +_fixate_caps (GstAggregator * agg, GstCaps * caps) { + GstVideoAggregator *vagg = GST_VIDEO_AGGREGATOR (agg); GList *l; gint best_width = -1, best_height = -1; gint best_fps_n = -1, best_fps_d = -1; @@ -945,21 +946,21 @@ _fixate_caps (GstVideoAggregator * vagg, GstCaps * caps) } static gboolean -_negotiated_caps (GstVideoAggregator * vagg, GstCaps * caps) +_negotiated_caps (GstAggregator * agg, GstCaps * caps) { GstVideoInfo v_info; - GST_DEBUG_OBJECT (vagg, "Negotiated caps %" GST_PTR_FORMAT, caps); + GST_DEBUG_OBJECT (agg, "Negotiated caps %" GST_PTR_FORMAT, caps); if (!gst_video_info_from_caps (&v_info, caps)) return FALSE; - if (!set_functions (GST_COMPOSITOR (vagg), &v_info)) { - GST_ERROR_OBJECT (vagg, "Failed to setup vfuncs"); + if (!set_functions (GST_COMPOSITOR (agg), &v_info)) { + GST_ERROR_OBJECT (agg, "Failed to setup vfuncs"); return FALSE; } - return TRUE; + return GST_AGGREGATOR_CLASS (parent_class)->negotiated_src_caps (agg, caps); } static GstFlowReturn @@ -1090,8 +1091,8 @@ gst_compositor_class_init (GstCompositorClass * klass) agg_class->sinkpads_type = GST_TYPE_COMPOSITOR_PAD; agg_class->sink_query = _sink_query; - videoaggregator_class->fixate_caps = _fixate_caps; - videoaggregator_class->negotiated_caps = _negotiated_caps; + agg_class->fixate_src_caps = _fixate_caps; + agg_class->negotiated_src_caps = _negotiated_caps; videoaggregator_class->aggregate_frames = gst_compositor_aggregate_frames; g_object_class_install_property (gobject_class, PROP_BACKGROUND, From b488a560ed80dcba11d79a2968e3870a47b9d30c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Olivier=20Cr=C3=AAte?= Date: Sun, 21 May 2017 12:41:53 +0200 Subject: [PATCH 297/381] videoaggregator: Return to parent on reconfigure The caps negotiation is now in the parent, so need to return there if a reconfiguration is needed, otherwise it will loops forever. --- gst-libs/gst/video/gstvideoaggregator.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index 7cdce00fa6..c731f92f1a 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -1445,7 +1445,6 @@ gst_video_aggregator_aggregate (GstAggregator * agg, gboolean timeout) GST_VIDEO_AGGREGATOR_LOCK (vagg); -restart: if (GST_VIDEO_INFO_FORMAT (&vagg->info) == GST_VIDEO_FORMAT_UNKNOWN) { if (timeout) gst_video_aggregator_advance_on_timeout (vagg); @@ -1510,7 +1509,8 @@ restart: */ if (gst_pad_needs_reconfigure (GST_AGGREGATOR_SRC_PAD (vagg))) { GST_DEBUG_OBJECT (vagg, "Need reconfigure"); - goto restart; + flow_ret = GST_AGGREGATOR_FLOW_NEED_DATA; + goto unlock_and_return; } GST_DEBUG_OBJECT (vagg, From 4a571db4c715ec200b23a92b82e84a94f8ffc4c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Olivier=20Cr=C3=AAte?= Date: Sat, 20 May 2017 17:25:16 +0200 Subject: [PATCH 298/381] videoaggregator: Get the buffer from the pool if available https://bugzilla.gnome.org/show_bug.cgi?id=746529 --- ext/gl/gstglbasemixer.c | 14 ------ ext/gl/gstglbasemixer.h | 2 - ext/gl/gstglmixer.c | 31 ------------ gst-libs/gst/video/gstvideoaggregator.c | 65 +++++++++++++++++++++---- 4 files changed, 56 insertions(+), 56 deletions(-) diff --git a/ext/gl/gstglbasemixer.c b/ext/gl/gstglbasemixer.c index 317d339ed4..d37e55c96a 100644 --- a/ext/gl/gstglbasemixer.c +++ b/ext/gl/gstglbasemixer.c @@ -580,20 +580,6 @@ no_decide_allocation: } } -GstBufferPool * -gst_gl_base_mixer_get_buffer_pool (GstGLBaseMixer * mix) -{ - GstBufferPool *pool; - - GST_OBJECT_LOCK (mix); - pool = mix->priv->pool; - if (pool) - gst_object_ref (pool); - GST_OBJECT_UNLOCK (mix); - - return pool; -} - static void gst_gl_base_mixer_get_property (GObject * object, guint prop_id, GValue * value, GParamSpec * pspec) diff --git a/ext/gl/gstglbasemixer.h b/ext/gl/gstglbasemixer.h index cba8184792..91bd8ef902 100644 --- a/ext/gl/gstglbasemixer.h +++ b/ext/gl/gstglbasemixer.h @@ -99,7 +99,5 @@ struct _GstGLBaseMixerClass GType gst_gl_base_mixer_get_type(void); -GstBufferPool *gst_gl_base_mixer_get_buffer_pool (GstGLBaseMixer * mix); - G_END_DECLS #endif /* __GST_GL_BASE_MIXER_H__ */ diff --git a/ext/gl/gstglmixer.c b/ext/gl/gstglmixer.c index d2ce49da8a..f1aa0b1028 100644 --- a/ext/gl/gstglmixer.c +++ b/ext/gl/gstglmixer.c @@ -322,8 +322,6 @@ static GstStaticPadTemplate sink_factory = GST_STATIC_PAD_TEMPLATE ("sink_%u", ); static gboolean gst_gl_mixer_src_query (GstAggregator * agg, GstQuery * query); -static GstFlowReturn gst_gl_mixer_get_output_buffer (GstVideoAggregator * - videoaggregator, GstBuffer ** outbuf); static gboolean gst_gl_mixer_stop (GstAggregator * agg); static gboolean gst_gl_mixer_start (GstAggregator * agg); @@ -371,7 +369,6 @@ gst_gl_mixer_class_init (GstGLMixerClass * klass) agg_class->negotiated_src_caps = _negotiated_caps; videoaggregator_class->aggregate_frames = gst_gl_mixer_aggregate_frames; - videoaggregator_class->get_output_buffer = gst_gl_mixer_get_output_buffer; videoaggregator_class->find_best_format = _find_best_format; mix_class->propose_allocation = gst_gl_mixer_propose_allocation; @@ -465,34 +462,6 @@ gst_gl_mixer_src_query (GstAggregator * agg, GstQuery * query) return res; } -static GstFlowReturn -gst_gl_mixer_get_output_buffer (GstVideoAggregator * videoaggregator, - GstBuffer ** outbuf) -{ - GstGLMixer *mix = GST_GL_MIXER (videoaggregator); - GstBufferPool *pool; - GstFlowReturn ret; - - pool = - gst_gl_base_mixer_get_buffer_pool (GST_GL_BASE_MIXER (videoaggregator)); - - if (!pool) - return GST_FLOW_NOT_NEGOTIATED; - - if (!gst_buffer_pool_is_active (pool)) { - if (!gst_buffer_pool_set_active (pool, TRUE)) { - GST_ELEMENT_ERROR (mix, RESOURCE, SETTINGS, - ("failed to activate bufferpool"), ("failed to activate bufferpool")); - return GST_FLOW_ERROR; - } - } - - ret = gst_buffer_pool_acquire_buffer (pool, outbuf, NULL); - gst_object_unref (pool); - - return ret; -} - static void _mixer_create_fbo (GstGLContext * context, GstGLMixer * mix) { diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index c731f92f1a..88f9a69a8b 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -1897,22 +1897,68 @@ gst_video_aggregator_release_pad (GstElement * element, GstPad * pad) return; } +static gboolean +gst_video_aggregator_decide_allocation (GstAggregator * self, GstQuery * query) +{ + GstAllocationParams params = { 0, 15, 0, 0 }; + guint i; + + if (gst_query_get_n_allocation_params (query) == 0) + gst_query_add_allocation_param (query, NULL, ¶ms); + else + for (i = 0; i < gst_query_get_n_allocation_params (query); i++) { + GstAllocator *allocator; + + gst_query_parse_nth_allocation_param (query, i, &allocator, ¶ms); + params.align = MAX (params.align, 15); + gst_query_set_nth_allocation_param (query, i, allocator, ¶ms); + } + + return TRUE; +} + static GstFlowReturn gst_video_aggregator_get_output_buffer (GstVideoAggregator * videoaggregator, GstBuffer ** outbuf) { - guint outsize; - static GstAllocationParams params = { 0, 15, 0, 0, }; + GstAggregator *aggregator = GST_AGGREGATOR (videoaggregator); + GstBufferPool *pool; + GstFlowReturn ret = GST_FLOW_OK; - outsize = GST_VIDEO_INFO_SIZE (&videoaggregator->info); - *outbuf = gst_buffer_new_allocate (NULL, outsize, ¶ms); + pool = gst_aggregator_get_buffer_pool (aggregator); - if (*outbuf == NULL) { - GST_ERROR_OBJECT (videoaggregator, - "Could not instantiate buffer of size: %d", outsize); + if (pool) { + if (!gst_buffer_pool_is_active (pool)) { + if (!gst_buffer_pool_set_active (pool, TRUE)) { + GST_ELEMENT_ERROR (videoaggregator, RESOURCE, SETTINGS, + ("failed to activate bufferpool"), + ("failed to activate bufferpool")); + return GST_FLOW_ERROR; + } + } + + ret = gst_buffer_pool_acquire_buffer (pool, outbuf, NULL); + gst_object_unref (pool); + } else { + guint outsize; + GstAllocator *allocator; + GstAllocationParams params; + + gst_aggregator_get_allocator (aggregator, &allocator, ¶ms); + + outsize = GST_VIDEO_INFO_SIZE (&videoaggregator->info); + *outbuf = gst_buffer_new_allocate (allocator, outsize, ¶ms); + + if (allocator) + gst_object_unref (allocator); + + if (*outbuf == NULL) { + GST_ELEMENT_ERROR (videoaggregator, RESOURCE, NO_SPACE_LEFT, + (NULL), ("Could not acquire buffer of size: %d", outsize)); + ret = GST_FLOW_ERROR; + } } - - return GST_FLOW_OK; + return ret; } static gboolean @@ -2090,6 +2136,7 @@ gst_video_aggregator_class_init (GstVideoAggregatorClass * klass) agg_class->fixate_src_caps = gst_video_aggregator_default_fixate_src_caps; agg_class->negotiated_src_caps = gst_video_aggregator_default_negotiated_src_caps; + agg_class->decide_allocation = gst_video_aggregator_decide_allocation; klass->find_best_format = gst_video_aggregator_find_best_format; klass->get_output_buffer = gst_video_aggregator_get_output_buffer; From d456da5cbfd036188c101b349e6cbb1e333a829b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Olivier=20Cr=C3=AAte?= Date: Sat, 20 May 2017 17:30:06 +0200 Subject: [PATCH 299/381] glbasemixer: Use aggregator for allocation handling https://bugzilla.gnome.org/show_bug.cgi?id=746529 --- ext/gl/gstglbasemixer.c | 121 ++-------------------------------------- 1 file changed, 4 insertions(+), 117 deletions(-) diff --git a/ext/gl/gstglbasemixer.c b/ext/gl/gstglbasemixer.c index d37e55c96a..afa16a843c 100644 --- a/ext/gl/gstglbasemixer.c +++ b/ext/gl/gstglbasemixer.c @@ -31,8 +31,6 @@ #define gst_gl_base_mixer_parent_class parent_class G_DEFINE_ABSTRACT_TYPE (GstGLBaseMixer, gst_gl_base_mixer, GST_TYPE_VIDEO_AGGREGATOR); -static gboolean gst_gl_base_mixer_do_bufferpool (GstGLBaseMixer * mix, - GstCaps * outcaps); #define GST_CAT_DEFAULT gst_gl_base_mixer_debug GST_DEBUG_CATEGORY (gst_gl_base_mixer_debug); @@ -107,17 +105,6 @@ gst_gl_base_mixer_pad_set_property (GObject * object, guint prop_id, } } -static gboolean -_negotiated_caps (GstAggregator * agg, GstCaps * caps) -{ - GstGLBaseMixer *mix = GST_GL_BASE_MIXER (agg); - - if (!gst_gl_base_mixer_do_bufferpool (mix, caps)) - return FALSE; - - return GST_AGGREGATOR_CLASS (parent_class)->negotiated_src_caps (agg, caps); -} - static gboolean _default_propose_allocation (GstGLBaseMixer * mix, GstGLBaseMixerPad * pad, GstQuery * decide_query, GstQuery * query) @@ -316,11 +303,8 @@ static void gst_gl_base_mixer_set_property (GObject * object, guint prop_id, static void gst_gl_base_mixer_get_property (GObject * object, guint prop_id, GValue * value, GParamSpec * pspec); -static gboolean gst_gl_base_mixer_decide_allocation (GstGLBaseMixer * mix, +static gboolean gst_gl_base_mixer_decide_allocation (GstAggregator * agg, GstQuery * query); -static gboolean gst_gl_base_mixer_set_allocation (GstGLBaseMixer * mix, - GstBufferPool * pool, GstAllocator * allocator, - GstAllocationParams * params, GstQuery * query); static void gst_gl_base_mixer_class_init (GstGLBaseMixerClass * klass) @@ -350,7 +334,7 @@ gst_gl_base_mixer_class_init (GstGLBaseMixerClass * klass) agg_class->src_activate = gst_gl_base_mixer_src_activate_mode; agg_class->stop = gst_gl_base_mixer_stop; agg_class->start = gst_gl_base_mixer_start; - agg_class->negotiated_src_caps = _negotiated_caps; + agg_class->decide_allocation = gst_gl_base_mixer_decide_allocation; klass->propose_allocation = _default_propose_allocation; @@ -468,8 +452,9 @@ gst_gl_base_mixer_src_query (GstAggregator * agg, GstQuery * query) } static gboolean -gst_gl_base_mixer_decide_allocation (GstGLBaseMixer * mix, GstQuery * query) +gst_gl_base_mixer_decide_allocation (GstAggregator * agg, GstQuery * query) { + GstGLBaseMixer *mix = GST_GL_BASE_MIXER (agg); GstGLBaseMixerClass *mix_class = GST_GL_BASE_MIXER_GET_CLASS (mix); if (!_get_gl_context (mix)) @@ -482,104 +467,6 @@ gst_gl_base_mixer_decide_allocation (GstGLBaseMixer * mix, GstQuery * query) return TRUE; } -/* takes ownership of the pool, allocator and query */ -static gboolean -gst_gl_base_mixer_set_allocation (GstGLBaseMixer * mix, - GstBufferPool * pool, GstAllocator * allocator, - GstAllocationParams * params, GstQuery * query) -{ - GstAllocator *oldalloc; - GstBufferPool *oldpool; - GstQuery *oldquery; - GstGLBaseMixerPrivate *priv = mix->priv; - - GST_DEBUG ("storing allocation query"); - - GST_OBJECT_LOCK (mix); - oldpool = priv->pool; - priv->pool = pool; - - oldalloc = priv->allocator; - priv->allocator = allocator; - - oldquery = priv->query; - priv->query = query; - - if (params) - priv->params = *params; - else - gst_allocation_params_init (&priv->params); - GST_OBJECT_UNLOCK (mix); - - if (oldpool) { - GST_DEBUG_OBJECT (mix, "deactivating old pool %p", oldpool); - gst_buffer_pool_set_active (oldpool, FALSE); - gst_object_unref (oldpool); - } - if (oldalloc) { - gst_object_unref (oldalloc); - } - if (oldquery) { - gst_query_unref (oldquery); - } - return TRUE; -} - -static gboolean -gst_gl_base_mixer_do_bufferpool (GstGLBaseMixer * mix, GstCaps * outcaps) -{ - GstQuery *query; - gboolean result = TRUE; - GstBufferPool *pool = NULL; - GstAllocator *allocator; - GstAllocationParams params; - GstAggregator *agg = GST_AGGREGATOR (mix); - - /* find a pool for the negotiated caps now */ - GST_DEBUG_OBJECT (mix, "doing allocation query"); - query = gst_query_new_allocation (outcaps, TRUE); - if (!gst_pad_peer_query (agg->srcpad, query)) { - /* not a problem, just debug a little */ - GST_DEBUG_OBJECT (mix, "peer ALLOCATION query failed"); - } - - GST_DEBUG_OBJECT (mix, "calling decide_allocation"); - result = gst_gl_base_mixer_decide_allocation (mix, query); - - GST_DEBUG_OBJECT (mix, "ALLOCATION (%d) params: %" GST_PTR_FORMAT, result, - query); - - if (!result) - goto no_decide_allocation; - - /* we got configuration from our peer or the decide_allocation method, - * parse them */ - if (gst_query_get_n_allocation_params (query) > 0) { - gst_query_parse_nth_allocation_param (query, 0, &allocator, ¶ms); - } else { - allocator = NULL; - gst_allocation_params_init (¶ms); - } - - if (gst_query_get_n_allocation_pools (query) > 0) - gst_query_parse_nth_allocation_pool (query, 0, &pool, NULL, NULL, NULL); - - /* now store */ - result = - gst_gl_base_mixer_set_allocation (mix, pool, allocator, ¶ms, query); - - return result; - - /* Errors */ -no_decide_allocation: - { - GST_WARNING_OBJECT (mix, "Failed to decide allocation"); - gst_query_unref (query); - - return result; - } -} - static void gst_gl_base_mixer_get_property (GObject * object, guint prop_id, GValue * value, GParamSpec * pspec) From b2d4a6835b8f030c7c47609d5458a9933340ebf6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Olivier=20Cr=C3=AAte?= Date: Sat, 20 May 2017 17:35:43 +0200 Subject: [PATCH 300/381] glbasemixer: Remove own decide_allocation, use GstAggregator's https://bugzilla.gnome.org/show_bug.cgi?id=746529 --- ext/gl/gstglbasemixer.c | 5 ----- ext/gl/gstglbasemixer.h | 1 - ext/gl/gstglmixer.c | 15 +++++++++++---- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/ext/gl/gstglbasemixer.c b/ext/gl/gstglbasemixer.c index afa16a843c..516dd1f16f 100644 --- a/ext/gl/gstglbasemixer.c +++ b/ext/gl/gstglbasemixer.c @@ -455,15 +455,10 @@ static gboolean gst_gl_base_mixer_decide_allocation (GstAggregator * agg, GstQuery * query) { GstGLBaseMixer *mix = GST_GL_BASE_MIXER (agg); - GstGLBaseMixerClass *mix_class = GST_GL_BASE_MIXER_GET_CLASS (mix); if (!_get_gl_context (mix)) return FALSE; - if (mix_class->decide_allocation) - if (!mix_class->decide_allocation (mix, query)) - return FALSE; - return TRUE; } diff --git a/ext/gl/gstglbasemixer.h b/ext/gl/gstglbasemixer.h index 91bd8ef902..087b634d46 100644 --- a/ext/gl/gstglbasemixer.h +++ b/ext/gl/gstglbasemixer.h @@ -92,7 +92,6 @@ struct _GstGLBaseMixerClass GstGLAPI supported_gl_api; gboolean (*propose_allocation) (GstGLBaseMixer * mix, GstGLBaseMixerPad * pad, GstQuery * decide_query, GstQuery *query); - gboolean (*decide_allocation) (GstGLBaseMixer * mix, GstQuery * decide_query); gpointer _padding[GST_PADDING]; }; diff --git a/ext/gl/gstglmixer.c b/ext/gl/gstglmixer.c index f1aa0b1028..cdf76269b8 100644 --- a/ext/gl/gstglmixer.c +++ b/ext/gl/gstglmixer.c @@ -334,7 +334,7 @@ static void gst_gl_mixer_set_property (GObject * object, guint prop_id, static void gst_gl_mixer_get_property (GObject * object, guint prop_id, GValue * value, GParamSpec * pspec); -static gboolean gst_gl_mixer_decide_allocation (GstGLBaseMixer * mix, +static gboolean gst_gl_mixer_decide_allocation (GstAggregator * agg, GstQuery * query); static void gst_gl_mixer_finalize (GObject * object); @@ -367,12 +367,12 @@ gst_gl_mixer_class_init (GstGLMixerClass * klass) agg_class->stop = gst_gl_mixer_stop; agg_class->start = gst_gl_mixer_start; agg_class->negotiated_src_caps = _negotiated_caps; + agg_class->decide_allocation = gst_gl_mixer_decide_allocation; videoaggregator_class->aggregate_frames = gst_gl_mixer_aggregate_frames; videoaggregator_class->find_best_format = _find_best_format; mix_class->propose_allocation = gst_gl_mixer_propose_allocation; - mix_class->decide_allocation = gst_gl_mixer_decide_allocation; /* Register the pad class */ g_type_class_ref (GST_TYPE_GL_MIXER_PAD); @@ -475,17 +475,24 @@ _mixer_create_fbo (GstGLContext * context, GstGLMixer * mix) } static gboolean -gst_gl_mixer_decide_allocation (GstGLBaseMixer * base_mix, GstQuery * query) +gst_gl_mixer_decide_allocation (GstAggregator * agg, GstQuery * query) { + GstGLBaseMixer *base_mix = GST_GL_BASE_MIXER (agg); GstGLMixer *mix = GST_GL_MIXER (base_mix); GstGLMixerClass *mixer_class = GST_GL_MIXER_GET_CLASS (mix); - GstGLContext *context = base_mix->context; + GstGLContext *context; GstBufferPool *pool = NULL; GstStructure *config; GstCaps *caps; guint min, max, size; gboolean update_pool; + if (!GST_AGGREGATOR_CLASS (gst_gl_mixer_parent_class)->decide_allocation (agg, + query)) + return FALSE; + + context = base_mix->context; + g_mutex_lock (&mix->priv->gl_resource_lock); mix->priv->gl_resource_ready = FALSE; if (mix->fbo) From af28bcab138d159c36938d2d0a68deea5065acfd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Olivier=20Cr=C3=AAte?= Date: Sat, 20 May 2017 18:10:29 +0200 Subject: [PATCH 301/381] videoaggregator: Create normal video pool as a fallback https://bugzilla.gnome.org/show_bug.cgi?id=746529 --- gst-libs/gst/video/gstvideoaggregator.c | 45 ++++++++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index 88f9a69a8b..2b3cf6fb93 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -1898,10 +1898,16 @@ gst_video_aggregator_release_pad (GstElement * element, GstPad * pad) } static gboolean -gst_video_aggregator_decide_allocation (GstAggregator * self, GstQuery * query) +gst_video_aggregator_decide_allocation (GstAggregator * agg, GstQuery * query) { + GstVideoAggregator *vagg = GST_VIDEO_AGGREGATOR (agg); GstAllocationParams params = { 0, 15, 0, 0 }; guint i; + GstBufferPool *pool; + guint size, min, max; + gboolean update = FALSE; + GstStructure *config = NULL; + GstCaps *caps = NULL; if (gst_query_get_n_allocation_params (query) == 0) gst_query_add_allocation_param (query, NULL, ¶ms); @@ -1914,6 +1920,43 @@ gst_video_aggregator_decide_allocation (GstAggregator * self, GstQuery * query) gst_query_set_nth_allocation_param (query, i, allocator, ¶ms); } + if (gst_query_get_n_allocation_pools (query) > 0) { + gst_query_parse_nth_allocation_pool (query, 0, &pool, &size, &min, &max); + + /* adjust size */ + size = MAX (size, vagg->info.size); + update = TRUE; + } else { + pool = NULL; + size = vagg->info.size; + min = max = 0; + update = FALSE; + } + + /* no downstream pool, make our own */ + if (pool == NULL) + pool = gst_video_buffer_pool_new (); + + config = gst_buffer_pool_get_config (pool); + + gst_query_parse_allocation (query, &caps, NULL); + if (caps) + gst_buffer_pool_config_set_params (config, caps, size, min, max); + + if (gst_query_find_allocation_meta (query, GST_VIDEO_META_API_TYPE, NULL)) { + gst_buffer_pool_config_add_option (config, + GST_BUFFER_POOL_OPTION_VIDEO_META); + } + gst_buffer_pool_set_config (pool, config); + + if (update) + gst_query_set_nth_allocation_pool (query, 0, pool, size, min, max); + else + gst_query_add_allocation_pool (query, pool, size, min, max); + + if (pool) + gst_object_unref (pool); + return TRUE; } From a2b0f2771a9045e777ad16ea24cfd5fcc5608e07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Olivier=20Cr=C3=AAte?= Date: Sun, 21 May 2017 15:30:10 +0200 Subject: [PATCH 302/381] gl*mixer: Use propose_allocation from the GstAggregator base class https://bugzilla.gnome.org/show_bug.cgi?id=782918 --- ext/gl/gstglbasemixer.c | 75 +++++++--------------------------------- ext/gl/gstglbasemixer.h | 2 -- ext/gl/gstglmixer.c | 19 ++++++---- ext/gl/gstglvideomixer.c | 18 +++++----- 4 files changed, 34 insertions(+), 80 deletions(-) diff --git a/ext/gl/gstglbasemixer.c b/ext/gl/gstglbasemixer.c index 516dd1f16f..e719abf77c 100644 --- a/ext/gl/gstglbasemixer.c +++ b/ext/gl/gstglbasemixer.c @@ -58,11 +58,6 @@ struct _GstGLBaseMixerPrivate gboolean negotiated; GstGLContext *other_context; - - GstBufferPool *pool; - GstAllocator *allocator; - GstAllocationParams params; - GstQuery *query; }; G_DEFINE_TYPE (GstGLBaseMixerPad, gst_gl_base_mixer_pad, @@ -105,13 +100,6 @@ gst_gl_base_mixer_pad_set_property (GObject * object, guint prop_id, } } -static gboolean -_default_propose_allocation (GstGLBaseMixer * mix, GstGLBaseMixerPad * pad, - GstQuery * decide_query, GstQuery * query) -{ - return TRUE; -} - static gboolean gst_gl_base_mixer_sink_event (GstAggregator * agg, GstAggregatorPad * bpad, GstEvent * event) @@ -210,53 +198,27 @@ context_error: } } +static gboolean +gst_gl_base_mixer_propose_allocation (GstAggregator * agg, + GstAggregatorPad * aggpad, GstQuery * decide_query, GstQuery * query) +{ + GstGLBaseMixer *mix = GST_GL_BASE_MIXER (agg); + + if (!_get_gl_context (mix)) + return FALSE; + + return TRUE; +} + static gboolean gst_gl_base_mixer_sink_query (GstAggregator * agg, GstAggregatorPad * bpad, GstQuery * query) { - gboolean ret = FALSE; GstGLBaseMixer *mix = GST_GL_BASE_MIXER (agg); - GstGLBaseMixerClass *mix_class = GST_GL_BASE_MIXER_GET_CLASS (mix); - GstGLBaseMixerPad *pad = GST_GL_BASE_MIXER_PAD (bpad); GST_TRACE ("QUERY %" GST_PTR_FORMAT, query); switch (GST_QUERY_TYPE (query)) { - case GST_QUERY_ALLOCATION: - { - GstQuery *decide_query = NULL; - - GST_OBJECT_LOCK (mix); - if (G_UNLIKELY (!pad->negotiated)) { - GST_DEBUG_OBJECT (mix, - "not negotiated yet, can't answer ALLOCATION query"); - GST_OBJECT_UNLOCK (mix); - return FALSE; - } - - if ((decide_query = mix->priv->query)) - gst_query_ref (decide_query); - GST_OBJECT_UNLOCK (mix); - - if (!_get_gl_context (mix)) - return FALSE; - - GST_DEBUG_OBJECT (mix, - "calling propose allocation with query %" GST_PTR_FORMAT, - decide_query); - - /* pass the query to the propose_allocation vmethod if any */ - if (mix_class->propose_allocation) - ret = mix_class->propose_allocation (mix, pad, decide_query, query); - else - ret = FALSE; - - if (decide_query) - gst_query_unref (decide_query); - - GST_DEBUG_OBJECT (mix, "ALLOCATION ret %d, %" GST_PTR_FORMAT, ret, query); - return ret; - } case GST_QUERY_CONTEXT: { if (gst_gl_handle_context_query ((GstElement *) mix, query, @@ -335,8 +297,7 @@ gst_gl_base_mixer_class_init (GstGLBaseMixerClass * klass) agg_class->stop = gst_gl_base_mixer_stop; agg_class->start = gst_gl_base_mixer_start; agg_class->decide_allocation = gst_gl_base_mixer_decide_allocation; - - klass->propose_allocation = _default_propose_allocation; + agg_class->propose_allocation = gst_gl_base_mixer_propose_allocation; g_object_class_install_property (gobject_class, PROP_CONTEXT, g_param_spec_object ("context", @@ -500,16 +461,6 @@ gst_gl_base_mixer_stop (GstAggregator * agg) { GstGLBaseMixer *mix = GST_GL_BASE_MIXER (agg); - if (mix->priv->query) { - gst_query_unref (mix->priv->query); - mix->priv->query = NULL; - } - - if (mix->priv->pool) { - gst_object_unref (mix->priv->pool); - mix->priv->pool = NULL; - } - if (mix->context) { gst_object_unref (mix->context); mix->context = NULL; diff --git a/ext/gl/gstglbasemixer.h b/ext/gl/gstglbasemixer.h index 087b634d46..35684087c2 100644 --- a/ext/gl/gstglbasemixer.h +++ b/ext/gl/gstglbasemixer.h @@ -91,8 +91,6 @@ struct _GstGLBaseMixerClass GstVideoAggregatorClass parent_class; GstGLAPI supported_gl_api; - gboolean (*propose_allocation) (GstGLBaseMixer * mix, GstGLBaseMixerPad * pad, GstQuery * decide_query, GstQuery *query); - gpointer _padding[GST_PADDING]; }; diff --git a/ext/gl/gstglmixer.c b/ext/gl/gstglmixer.c index cdf76269b8..c0486616b2 100644 --- a/ext/gl/gstglmixer.c +++ b/ext/gl/gstglmixer.c @@ -131,17 +131,25 @@ _find_best_format (GstVideoAggregator * vagg, GstCaps * downstream_caps, } static gboolean -gst_gl_mixer_propose_allocation (GstGLBaseMixer * base_mix, - GstGLBaseMixerPad * base_pad, GstQuery * decide_query, GstQuery * query) +gst_gl_mixer_propose_allocation (GstAggregator * agg, + GstAggregatorPad * agg_pad, GstQuery * decide_query, GstQuery * query) { - GstGLMixer *mix = GST_GL_MIXER (base_mix); - GstGLContext *context = base_mix->context; + GstGLMixer *mix = GST_GL_MIXER (agg); + GstGLBaseMixer *base_mix = GST_GL_BASE_MIXER (agg); + GstGLContext *context; GstBufferPool *pool = NULL; GstStructure *config; GstCaps *caps; guint size = 0; gboolean need_pool; + + if (!GST_AGGREGATOR_CLASS (gst_gl_mixer_parent_class)->propose_allocation + (agg, agg_pad, decide_query, query)) + return FALSE; + + context = base_mix->context; + gst_query_parse_allocation (query, &caps, &need_pool); if (caps == NULL) @@ -347,7 +355,6 @@ gst_gl_mixer_class_init (GstGLMixerClass * klass) GstVideoAggregatorClass *videoaggregator_class = (GstVideoAggregatorClass *) klass; GstAggregatorClass *agg_class = (GstAggregatorClass *) klass; - GstGLBaseMixerClass *mix_class = GST_GL_BASE_MIXER_CLASS (klass);; GST_DEBUG_CATEGORY_INIT (GST_CAT_DEFAULT, "glmixer", 0, "OpenGL mixer"); @@ -368,11 +375,11 @@ gst_gl_mixer_class_init (GstGLMixerClass * klass) agg_class->start = gst_gl_mixer_start; agg_class->negotiated_src_caps = _negotiated_caps; agg_class->decide_allocation = gst_gl_mixer_decide_allocation; + agg_class->propose_allocation = gst_gl_mixer_propose_allocation; videoaggregator_class->aggregate_frames = gst_gl_mixer_aggregate_frames; videoaggregator_class->find_best_format = _find_best_format; - mix_class->propose_allocation = gst_gl_mixer_propose_allocation; /* Register the pad class */ g_type_class_ref (GST_TYPE_GL_MIXER_PAD); diff --git a/ext/gl/gstglvideomixer.c b/ext/gl/gstglvideomixer.c index 10251e6792..a5a094b67e 100644 --- a/ext/gl/gstglvideomixer.c +++ b/ext/gl/gstglvideomixer.c @@ -455,9 +455,8 @@ static void gst_gl_video_mixer_get_property (GObject * object, guint prop_id, static GstCaps *_update_caps (GstVideoAggregator * vagg, GstCaps * caps); static GstCaps *_fixate_caps (GstAggregator * agg, GstCaps * caps); -static gboolean gst_gl_video_mixer_propose_allocation (GstGLBaseMixer * - base_mix, GstGLBaseMixerPad * base_pad, GstQuery * decide_query, - GstQuery * query); +static gboolean gst_gl_video_mixer_propose_allocation (GstAggregator * + agg, GstAggregatorPad * agg_pad, GstQuery * decide_query, GstQuery * query); static void gst_gl_video_mixer_reset (GstGLMixer * mixer); static gboolean gst_gl_video_mixer_init_shader (GstGLMixer * mixer, GstCaps * outcaps); @@ -849,7 +848,6 @@ gst_gl_video_mixer_class_init (GstGLVideoMixerClass * klass) GstElementClass *element_class; GstAggregatorClass *agg_class = (GstAggregatorClass *) klass; GstVideoAggregatorClass *vagg_class = (GstVideoAggregatorClass *) klass; - GstGLBaseMixerClass *mix_class = GST_GL_BASE_MIXER_CLASS (klass); gobject_class = (GObjectClass *) klass; element_class = GST_ELEMENT_CLASS (klass); @@ -872,12 +870,12 @@ gst_gl_video_mixer_class_init (GstGLVideoMixerClass * klass) GST_GL_MIXER_CLASS (klass)->process_textures = gst_gl_video_mixer_process_textures; + vagg_class->update_caps = _update_caps; agg_class->sinkpads_type = GST_TYPE_GL_VIDEO_MIXER_PAD; agg_class->fixate_src_caps = _fixate_caps; - - mix_class->propose_allocation = gst_gl_video_mixer_propose_allocation; + agg_class->propose_allocation = gst_gl_video_mixer_propose_allocation; GST_GL_BASE_MIXER_CLASS (klass)->supported_gl_api = GST_GL_API_OPENGL | GST_GL_API_OPENGL3 | GST_GL_API_GLES2; @@ -923,11 +921,11 @@ gst_gl_video_mixer_get_property (GObject * object, guint prop_id, } static gboolean -gst_gl_video_mixer_propose_allocation (GstGLBaseMixer * base_mix, - GstGLBaseMixerPad * base_pad, GstQuery * decide_query, GstQuery * query) +gst_gl_video_mixer_propose_allocation (GstAggregator * agg, + GstAggregatorPad * agg_pad, GstQuery * decide_query, GstQuery * query) { - if (!GST_GL_BASE_MIXER_CLASS (parent_class)->propose_allocation (base_mix, - base_pad, decide_query, query)) + if (!GST_AGGREGATOR_CLASS (parent_class)->propose_allocation (agg, + agg_pad, decide_query, query)) return FALSE; gst_query_add_allocation_meta (query, From da4913f2f6a7f42ba3bb0b1cfcca94c924b93211 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Olivier=20Cr=C3=AAte?= Date: Sun, 21 May 2017 15:44:02 +0200 Subject: [PATCH 303/381] videoaggregator: Declare that it supports the video meta on input https://bugzilla.gnome.org/show_bug.cgi?id=782918 --- gst-libs/gst/video/gstvideoaggregator.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index 2b3cf6fb93..11a3ef40c4 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -1897,6 +1897,15 @@ gst_video_aggregator_release_pad (GstElement * element, GstPad * pad) return; } +static gboolean +gst_video_aggregator_propose_allocation (GstAggregator * agg, + GstAggregatorPad * pad, GstQuery * decide_query, GstQuery * query) +{ + gst_query_add_allocation_meta (query, GST_VIDEO_META_API_TYPE, NULL); + + return TRUE; +} + static gboolean gst_video_aggregator_decide_allocation (GstAggregator * agg, GstQuery * query) { @@ -2180,6 +2189,7 @@ gst_video_aggregator_class_init (GstVideoAggregatorClass * klass) agg_class->negotiated_src_caps = gst_video_aggregator_default_negotiated_src_caps; agg_class->decide_allocation = gst_video_aggregator_decide_allocation; + agg_class->propose_allocation = gst_video_aggregator_propose_allocation; klass->find_best_format = gst_video_aggregator_find_best_format; klass->get_output_buffer = gst_video_aggregator_get_output_buffer; From 1dc6f66310acb673f791cdad4243d126e8b0175d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Olivier=20Cr=C3=AAte?= Date: Sun, 21 May 2017 18:31:59 +0200 Subject: [PATCH 304/381] glbasemixer: Remove unused negotiated member This is now all handled in GstAggregator, so this code is not called anymore. --- ext/gl/gstglbasemixer.c | 44 ----------------------------------------- ext/gl/gstglbasemixer.h | 2 -- 2 files changed, 46 deletions(-) diff --git a/ext/gl/gstglbasemixer.c b/ext/gl/gstglbasemixer.c index e719abf77c..3e77e3394d 100644 --- a/ext/gl/gstglbasemixer.c +++ b/ext/gl/gstglbasemixer.c @@ -100,25 +100,6 @@ gst_gl_base_mixer_pad_set_property (GObject * object, guint prop_id, } } -static gboolean -gst_gl_base_mixer_sink_event (GstAggregator * agg, GstAggregatorPad * bpad, - GstEvent * event) -{ - GstGLBaseMixerPad *pad = GST_GL_BASE_MIXER_PAD (bpad); - - switch (GST_EVENT_TYPE (event)) { - case GST_EVENT_CAPS: - if (!GST_AGGREGATOR_CLASS (parent_class)->sink_event (agg, bpad, event)) - return FALSE; - - pad->negotiated = TRUE; - return TRUE; - default: - break; - } - - return GST_AGGREGATOR_CLASS (parent_class)->sink_event (agg, bpad, event); -} static gboolean _find_local_gl_context (GstGLBaseMixer * mix) @@ -291,7 +272,6 @@ gst_gl_base_mixer_class_init (GstGLBaseMixerClass * klass) agg_class->sinkpads_type = GST_TYPE_GL_BASE_MIXER_PAD; agg_class->sink_query = gst_gl_base_mixer_sink_query; - agg_class->sink_event = gst_gl_base_mixer_sink_event; agg_class->src_query = gst_gl_base_mixer_src_query; agg_class->src_activate = gst_gl_base_mixer_src_activate_mode; agg_class->stop = gst_gl_base_mixer_stop; @@ -311,32 +291,10 @@ gst_gl_base_mixer_class_init (GstGLBaseMixerClass * klass) klass->supported_gl_api = GST_GL_API_ANY; } -static gboolean -_reset_pad (GstAggregator * self, GstAggregatorPad * base_pad, - gpointer user_data) -{ - GstGLBaseMixerPad *mix_pad = GST_GL_BASE_MIXER_PAD (base_pad); - - mix_pad->negotiated = FALSE; - - return TRUE; -} - -static void -gst_gl_base_mixer_reset (GstGLBaseMixer * mix) -{ - /* clean up collect data */ - - gst_aggregator_iterate_sinkpads (GST_AGGREGATOR (mix), - (GstAggregatorPadForeachFunc) _reset_pad, NULL); -} - static void gst_gl_base_mixer_init (GstGLBaseMixer * mix) { mix->priv = GST_GL_BASE_MIXER_GET_PRIVATE (mix); - - gst_gl_base_mixer_reset (mix); } static void @@ -466,8 +424,6 @@ gst_gl_base_mixer_stop (GstAggregator * agg) mix->context = NULL; } - gst_gl_base_mixer_reset (mix); - return TRUE; } diff --git a/ext/gl/gstglbasemixer.h b/ext/gl/gstglbasemixer.h index 35684087c2..e9684abc4d 100644 --- a/ext/gl/gstglbasemixer.h +++ b/ext/gl/gstglbasemixer.h @@ -47,8 +47,6 @@ typedef struct _GstGLBaseMixerPadClass GstGLBaseMixerPadClass; struct _GstGLBaseMixerPad { GstVideoAggregatorPad parent; /* subclass the pad */ - - gboolean negotiated; }; struct _GstGLBaseMixerPadClass From 027a62a9d178cc2233b6c7ca9723f5c9daa38f7a Mon Sep 17 00:00:00 2001 From: Mathieu Duponchelle Date: Tue, 11 Apr 2017 01:18:51 +0200 Subject: [PATCH 305/381] videoaggregator: fix gaps at end of streams. When the pad has received EOS, its buffer may still be mixed any number of times, when the pad's framerate is inferior to the output framerate. This was introduced by my patch in https://bugzilla.gnome.org/show_bug.cgi?id=782962, this patch also correctly addresses the initial issue. --- gst-libs/gst/video/gstvideoaggregator.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index 11a3ef40c4..0de0bc6c45 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -1219,16 +1219,16 @@ gst_video_aggregator_fill_queues (GstVideoAggregator * vagg, if (pad->priv->end_time != -1) { if (pad->priv->end_time <= output_start_running_time) { pad->priv->start_time = pad->priv->end_time = -1; - if (is_eos) { + if (!is_eos) { GST_DEBUG ("I just need more data"); need_more_data = TRUE; + } else { + gst_buffer_replace (&pad->buffer, NULL); } } else if (is_eos) { eos = FALSE; } - } - - if (is_eos) { + } else if (is_eos) { gst_buffer_replace (&pad->buffer, NULL); } } From d483df1d402dfe74ee800d74a1c1a4430a6b2e33 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim-Philipp=20M=C3=BCller?= Date: Tue, 18 Jul 2017 12:46:09 +0100 Subject: [PATCH 306/381] video: mark symbols explicitly for export with GST_EXPORT --- gst-libs/gst/video/gstvideoaggregator.h | 1 + 1 file changed, 1 insertion(+) diff --git a/gst-libs/gst/video/gstvideoaggregator.h b/gst-libs/gst/video/gstvideoaggregator.h index ced6124aec..3c74d1aa84 100644 --- a/gst-libs/gst/video/gstvideoaggregator.h +++ b/gst-libs/gst/video/gstvideoaggregator.h @@ -110,6 +110,7 @@ struct _GstVideoAggregatorClass gpointer _gst_reserved[GST_PADDING_LARGE]; }; +GST_EXPORT GType gst_video_aggregator_get_type (void); G_END_DECLS From effb593a9314f4ec2a20e6d4a9bdf40686bf16e5 Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Thu, 6 Jul 2017 14:26:21 -0400 Subject: [PATCH 307/381] compositor: Add support for crossfade blending Crossfading is a bit more complex than just having two pads with the right keyframes as the blending is not exactly the same. The difference is in the way we compute the alpha channel, in the case of crossfading, we have to compute an additive operation between the destination and the source (factored by the alpha property of both the input pad alpha property and the crossfading ratio) basically so that the crossfade result of 2 opaque frames is also fully opaque at any time in the crossfading process, avoid bleeding through the layer blending. Some rationnal can be found in https://phabricator.freedesktop.org/T7773. https://bugzilla.gnome.org/show_bug.cgi?id=784827 --- gst-libs/gst/video/gstvideoaggregator.c | 13 +- gst/compositor/blend.c | 145 ++++++++++-------- gst/compositor/blend.h | 24 ++- gst/compositor/compositor.c | 192 ++++++++++++++++++++---- gst/compositor/compositor.h | 5 +- gst/compositor/compositororc.orc | 144 +++++++++++++++++- gst/compositor/compositorpad.h | 3 + 7 files changed, 426 insertions(+), 100 deletions(-) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index 0de0bc6c45..9864612acf 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -521,7 +521,18 @@ gst_video_aggregator_find_best_format (GstVideoAggregator * vagg, GINT_TO_POINTER (format_number)); /* If that pad is the first with alpha, set it as the new best format */ - if (!need_alpha && (pad->info.finfo->flags & GST_VIDEO_FORMAT_FLAG_ALPHA)) { + if (!need_alpha && (pad->ABI.needs_alpha + && (!GST_VIDEO_FORMAT_INFO_HAS_ALPHA (pad->info.finfo)))) { + need_alpha = TRUE; + /* Just fallback to ARGB in case we require alpha but the input pad + * does not have alpha. + * Do not increment best_format_number in that case. */ + gst_video_info_set_format (best_info, + GST_VIDEO_FORMAT_ARGB, + GST_VIDEO_INFO_HEIGHT (&pad->info), + GST_VIDEO_INFO_WIDTH (&pad->info)); + } else if (!need_alpha + && (pad->info.finfo->flags & GST_VIDEO_FORMAT_FLAG_ALPHA)) { need_alpha = TRUE; *best_info = pad->info; best_format_number = format_number; diff --git a/gst/compositor/blend.c b/gst/compositor/blend.c index 721c766e68..b36c164654 100644 --- a/gst/compositor/blend.c +++ b/gst/compositor/blend.c @@ -44,7 +44,7 @@ GST_DEBUG_CATEGORY_STATIC (gst_compositor_blend_debug); #define BLEND_A32(name, method, LOOP) \ static void \ method##_ ##name (GstVideoFrame * srcframe, gint xpos, gint ypos, \ - gdouble src_alpha, GstVideoFrame * destframe) \ + gdouble src_alpha, GstVideoFrame * destframe, GstCompositorBlendMode mode) \ { \ guint s_alpha; \ gint src_stride, dest_stride; \ @@ -89,24 +89,45 @@ method##_ ##name (GstVideoFrame * srcframe, gint xpos, gint ypos, \ if (src_height > 0 && src_width > 0) { \ dest = dest + 4 * xpos + (ypos * dest_stride); \ \ - LOOP (dest, src, src_height, src_width, src_stride, dest_stride, s_alpha); \ + LOOP (dest, src, src_height, src_width, src_stride, dest_stride, s_alpha, \ + mode); \ } \ } -#define BLEND_A32_LOOP(name, method) \ +#define OVERLAY_A32_LOOP(name) \ static inline void \ -_##method##_loop_##name (guint8 * dest, const guint8 * src, gint src_height, \ - gint src_width, gint src_stride, gint dest_stride, guint s_alpha) \ +_overlay_loop_##name (guint8 * dest, const guint8 * src, gint src_height, \ + gint src_width, gint src_stride, gint dest_stride, guint s_alpha, \ + GstCompositorBlendMode mode) \ { \ s_alpha = MIN (255, s_alpha); \ - compositor_orc_##method##_##name (dest, dest_stride, src, src_stride, \ - s_alpha, src_width, src_height); \ + switch (mode) { \ + case COMPOSITOR_BLEND_MODE_NORMAL:\ + compositor_orc_overlay_##name (dest, dest_stride, src, src_stride, \ + s_alpha, src_width, src_height); \ + break;\ + case COMPOSITOR_BLEND_MODE_ADDITIVE:\ + compositor_orc_overlay_##name##_addition (dest, dest_stride, src, src_stride, \ + s_alpha, src_width, src_height); \ + break;\ + }\ } -BLEND_A32_LOOP (argb, blend); -BLEND_A32_LOOP (bgra, blend); -BLEND_A32_LOOP (argb, overlay); -BLEND_A32_LOOP (bgra, overlay); +#define BLEND_A32_LOOP_WITH_MODE(name) \ +static inline void \ +_blend_loop_##name (guint8 * dest, const guint8 * src, gint src_height, \ + gint src_width, gint src_stride, gint dest_stride, guint s_alpha, \ + GstCompositorBlendMode mode) \ +{ \ + s_alpha = MIN (255, s_alpha); \ + compositor_orc_blend_##name (dest, dest_stride, src, src_stride, \ + s_alpha, src_width, src_height); \ +} + +OVERLAY_A32_LOOP (argb); +OVERLAY_A32_LOOP (bgra); +BLEND_A32_LOOP_WITH_MODE (argb); +BLEND_A32_LOOP_WITH_MODE (bgra); #if G_BYTE_ORDER == G_LITTLE_ENDIAN BLEND_A32 (argb, blend, _blend_loop_argb); @@ -228,12 +249,12 @@ _blend_##format_name (const guint8 * src, guint8 * dest, \ \ b_alpha = CLAMP ((gint) (src_alpha * 256), 0, 256); \ \ - BLENDLOOP(dest, dest_stride, src, src_stride, b_alpha, src_width, src_height); \ + BLENDLOOP(dest, dest_stride, src, src_stride, b_alpha, src_width, src_height);\ } \ \ static void \ blend_##format_name (GstVideoFrame * srcframe, gint xpos, gint ypos, \ - gdouble src_alpha, GstVideoFrame * destframe) \ + gdouble src_alpha, GstVideoFrame * destframe, GstCompositorBlendMode mode) \ { \ const guint8 *b_src; \ guint8 *b_dest; \ @@ -478,7 +499,7 @@ _blend_##format_name (const guint8 * src, guint8 * dest, \ \ static void \ blend_##format_name (GstVideoFrame * srcframe, gint xpos, gint ypos, \ - gdouble src_alpha, GstVideoFrame * destframe) \ + gdouble src_alpha, GstVideoFrame * destframe, GstCompositorBlendMode mode) \ { \ const guint8 *b_src; \ guint8 *b_dest; \ @@ -649,7 +670,7 @@ NV_YUV_FILL_CHECKER (nv21, memset); #define RGB_BLEND(name, bpp, MEMCPY, BLENDLOOP) \ static void \ blend_##name (GstVideoFrame * srcframe, gint xpos, gint ypos, \ - gdouble src_alpha, GstVideoFrame * destframe) \ + gdouble src_alpha, GstVideoFrame * destframe, GstCompositorBlendMode mode) \ { \ gint b_alpha; \ gint i; \ @@ -815,7 +836,7 @@ RGB_FILL_COLOR (bgrx, 4, _memset_bgrx); #define PACKED_422_BLEND(name, MEMCPY, BLENDLOOP) \ static void \ blend_##name (GstVideoFrame * srcframe, gint xpos, gint ypos, \ - gdouble src_alpha, GstVideoFrame * destframe) \ + gdouble src_alpha, GstVideoFrame * destframe, GstCompositorBlendMode mode) \ { \ gint b_alpha; \ gint i; \ @@ -1010,52 +1031,52 @@ gst_compositor_init_blend (void) GST_DEBUG_CATEGORY_INIT (gst_compositor_blend_debug, "compositor_blend", 0, "video compositor blending functions"); - gst_compositor_blend_argb = blend_argb; - gst_compositor_blend_bgra = blend_bgra; - gst_compositor_overlay_argb = overlay_argb; - gst_compositor_overlay_bgra = overlay_bgra; - gst_compositor_blend_i420 = blend_i420; - gst_compositor_blend_nv12 = blend_nv12; - gst_compositor_blend_nv21 = blend_nv21; - gst_compositor_blend_y444 = blend_y444; - gst_compositor_blend_y42b = blend_y42b; - gst_compositor_blend_y41b = blend_y41b; - gst_compositor_blend_rgb = blend_rgb; - gst_compositor_blend_xrgb = blend_xrgb; - gst_compositor_blend_yuy2 = blend_yuy2; + gst_compositor_blend_argb = GST_DEBUG_FUNCPTR (blend_argb); + gst_compositor_blend_bgra = GST_DEBUG_FUNCPTR (blend_bgra); + gst_compositor_overlay_argb = GST_DEBUG_FUNCPTR (overlay_argb); + gst_compositor_overlay_bgra = GST_DEBUG_FUNCPTR (overlay_bgra); + gst_compositor_blend_i420 = GST_DEBUG_FUNCPTR (blend_i420); + gst_compositor_blend_nv12 = GST_DEBUG_FUNCPTR (blend_nv12); + gst_compositor_blend_nv21 = GST_DEBUG_FUNCPTR (blend_nv21); + gst_compositor_blend_y444 = GST_DEBUG_FUNCPTR (blend_y444); + gst_compositor_blend_y42b = GST_DEBUG_FUNCPTR (blend_y42b); + gst_compositor_blend_y41b = GST_DEBUG_FUNCPTR (blend_y41b); + gst_compositor_blend_rgb = GST_DEBUG_FUNCPTR (blend_rgb); + gst_compositor_blend_xrgb = GST_DEBUG_FUNCPTR (blend_xrgb); + gst_compositor_blend_yuy2 = GST_DEBUG_FUNCPTR (blend_yuy2); - gst_compositor_fill_checker_argb = fill_checker_argb_c; - gst_compositor_fill_checker_bgra = fill_checker_bgra_c; - gst_compositor_fill_checker_ayuv = fill_checker_ayuv_c; - gst_compositor_fill_checker_i420 = fill_checker_i420; - gst_compositor_fill_checker_nv12 = fill_checker_nv12; - gst_compositor_fill_checker_nv21 = fill_checker_nv21; - gst_compositor_fill_checker_y444 = fill_checker_y444; - gst_compositor_fill_checker_y42b = fill_checker_y42b; - gst_compositor_fill_checker_y41b = fill_checker_y41b; - gst_compositor_fill_checker_rgb = fill_checker_rgb_c; - gst_compositor_fill_checker_xrgb = fill_checker_xrgb_c; - gst_compositor_fill_checker_yuy2 = fill_checker_yuy2_c; - gst_compositor_fill_checker_uyvy = fill_checker_uyvy_c; + gst_compositor_fill_checker_argb = GST_DEBUG_FUNCPTR (fill_checker_argb_c); + gst_compositor_fill_checker_bgra = GST_DEBUG_FUNCPTR (fill_checker_bgra_c); + gst_compositor_fill_checker_ayuv = GST_DEBUG_FUNCPTR (fill_checker_ayuv_c); + gst_compositor_fill_checker_i420 = GST_DEBUG_FUNCPTR (fill_checker_i420); + gst_compositor_fill_checker_nv12 = GST_DEBUG_FUNCPTR (fill_checker_nv12); + gst_compositor_fill_checker_nv21 = GST_DEBUG_FUNCPTR (fill_checker_nv21); + gst_compositor_fill_checker_y444 = GST_DEBUG_FUNCPTR (fill_checker_y444); + gst_compositor_fill_checker_y42b = GST_DEBUG_FUNCPTR (fill_checker_y42b); + gst_compositor_fill_checker_y41b = GST_DEBUG_FUNCPTR (fill_checker_y41b); + gst_compositor_fill_checker_rgb = GST_DEBUG_FUNCPTR (fill_checker_rgb_c); + gst_compositor_fill_checker_xrgb = GST_DEBUG_FUNCPTR (fill_checker_xrgb_c); + gst_compositor_fill_checker_yuy2 = GST_DEBUG_FUNCPTR (fill_checker_yuy2_c); + gst_compositor_fill_checker_uyvy = GST_DEBUG_FUNCPTR (fill_checker_uyvy_c); - gst_compositor_fill_color_argb = fill_color_argb; - gst_compositor_fill_color_bgra = fill_color_bgra; - gst_compositor_fill_color_abgr = fill_color_abgr; - gst_compositor_fill_color_rgba = fill_color_rgba; - gst_compositor_fill_color_ayuv = fill_color_ayuv; - gst_compositor_fill_color_i420 = fill_color_i420; - gst_compositor_fill_color_yv12 = fill_color_yv12; - gst_compositor_fill_color_nv12 = fill_color_nv12; - gst_compositor_fill_color_y444 = fill_color_y444; - gst_compositor_fill_color_y42b = fill_color_y42b; - gst_compositor_fill_color_y41b = fill_color_y41b; - gst_compositor_fill_color_rgb = fill_color_rgb_c; - gst_compositor_fill_color_bgr = fill_color_bgr_c; - gst_compositor_fill_color_xrgb = fill_color_xrgb; - gst_compositor_fill_color_xbgr = fill_color_xbgr; - gst_compositor_fill_color_rgbx = fill_color_rgbx; - gst_compositor_fill_color_bgrx = fill_color_bgrx; - gst_compositor_fill_color_yuy2 = fill_color_yuy2; - gst_compositor_fill_color_yvyu = fill_color_yvyu; - gst_compositor_fill_color_uyvy = fill_color_uyvy; + gst_compositor_fill_color_argb = GST_DEBUG_FUNCPTR (fill_color_argb); + gst_compositor_fill_color_bgra = GST_DEBUG_FUNCPTR (fill_color_bgra); + gst_compositor_fill_color_abgr = GST_DEBUG_FUNCPTR (fill_color_abgr); + gst_compositor_fill_color_rgba = GST_DEBUG_FUNCPTR (fill_color_rgba); + gst_compositor_fill_color_ayuv = GST_DEBUG_FUNCPTR (fill_color_ayuv); + gst_compositor_fill_color_i420 = GST_DEBUG_FUNCPTR (fill_color_i420); + gst_compositor_fill_color_yv12 = GST_DEBUG_FUNCPTR (fill_color_yv12); + gst_compositor_fill_color_nv12 = GST_DEBUG_FUNCPTR (fill_color_nv12); + gst_compositor_fill_color_y444 = GST_DEBUG_FUNCPTR (fill_color_y444); + gst_compositor_fill_color_y42b = GST_DEBUG_FUNCPTR (fill_color_y42b); + gst_compositor_fill_color_y41b = GST_DEBUG_FUNCPTR (fill_color_y41b); + gst_compositor_fill_color_rgb = GST_DEBUG_FUNCPTR (fill_color_rgb_c); + gst_compositor_fill_color_bgr = GST_DEBUG_FUNCPTR (fill_color_bgr_c); + gst_compositor_fill_color_xrgb = GST_DEBUG_FUNCPTR (fill_color_xrgb); + gst_compositor_fill_color_xbgr = GST_DEBUG_FUNCPTR (fill_color_xbgr); + gst_compositor_fill_color_rgbx = GST_DEBUG_FUNCPTR (fill_color_rgbx); + gst_compositor_fill_color_bgrx = GST_DEBUG_FUNCPTR (fill_color_bgrx); + gst_compositor_fill_color_yuy2 = GST_DEBUG_FUNCPTR (fill_color_yuy2); + gst_compositor_fill_color_yvyu = GST_DEBUG_FUNCPTR (fill_color_yvyu); + gst_compositor_fill_color_uyvy = GST_DEBUG_FUNCPTR (fill_color_uyvy); } diff --git a/gst/compositor/blend.h b/gst/compositor/blend.h index 1cc127c4cf..34d52306a0 100644 --- a/gst/compositor/blend.h +++ b/gst/compositor/blend.h @@ -1,4 +1,4 @@ -/* +/* * Copyright (C) 2009 Sebastian Dröge * * This library is free software; you can redistribute it and/or @@ -23,7 +23,21 @@ #include #include -typedef void (*BlendFunction) (GstVideoFrame *srcframe, gint xpos, gint ypos, gdouble src_alpha, GstVideoFrame * destframe); +/** + * GstCompositorBlendMode: + * @COMPOSITOR_BLEND_MODE_NORMAL: Normal blending + * @COMPOSITOR_BLEND_MODE_ADDITIVE: Alphas are simply added, + * + * The different modes compositor can use for blending. + */ +typedef enum +{ + COMPOSITOR_BLEND_MODE_NORMAL, + COMPOSITOR_BLEND_MODE_ADDITIVE, +} GstCompositorBlendMode; + +typedef void (*BlendFunction) (GstVideoFrame *srcframe, gint xpos, gint ypos, gdouble src_alpha, GstVideoFrame * destframe, + GstCompositorBlendMode mode); typedef void (*FillCheckerFunction) (GstVideoFrame * frame); typedef void (*FillColorFunction) (GstVideoFrame * frame, gint c1, gint c2, gint c3); @@ -32,10 +46,16 @@ extern BlendFunction gst_compositor_blend_bgra; #define gst_compositor_blend_ayuv gst_compositor_blend_argb #define gst_compositor_blend_abgr gst_compositor_blend_argb #define gst_compositor_blend_rgba gst_compositor_blend_bgra +#define gst_compositor_blend_ayuv_addition gst_compositor_blend_argb_addition +#define gst_compositor_blend_abgr_addition gst_compositor_blend_argb_addition +#define gst_compositor_blend_rgba_addition gst_compositor_blend_bgra_addition + + extern BlendFunction gst_compositor_overlay_argb; extern BlendFunction gst_compositor_overlay_bgra; #define gst_compositor_overlay_ayuv gst_compositor_overlay_argb #define gst_compositor_overlay_abgr gst_compositor_overlay_argb +#define gst_compositor_overlay_abgr gst_compositor_overlay_argb #define gst_compositor_overlay_rgba gst_compositor_overlay_bgra extern BlendFunction gst_compositor_blend_i420; #define gst_compositor_blend_yv12 gst_compositor_blend_i420 diff --git a/gst/compositor/compositor.c b/gst/compositor/compositor.c index 23dc20cda1..dceca349c8 100644 --- a/gst/compositor/compositor.c +++ b/gst/compositor/compositor.c @@ -125,6 +125,7 @@ static GstStaticPadTemplate sink_factory = GST_STATIC_PAD_TEMPLATE ("sink_%u", #define DEFAULT_PAD_WIDTH 0 #define DEFAULT_PAD_HEIGHT 0 #define DEFAULT_PAD_ALPHA 1.0 +#define DEFAULT_PAD_CROSSFADE_RATIO -1.0 enum { PROP_PAD_0, @@ -132,7 +133,8 @@ enum PROP_PAD_YPOS, PROP_PAD_WIDTH, PROP_PAD_HEIGHT, - PROP_PAD_ALPHA + PROP_PAD_ALPHA, + PROP_PAD_CROSSFADE_RATIO, }; G_DEFINE_TYPE (GstCompositorPad, gst_compositor_pad, @@ -160,6 +162,9 @@ gst_compositor_pad_get_property (GObject * object, guint prop_id, case PROP_PAD_ALPHA: g_value_set_double (value, pad->alpha); break; + case PROP_PAD_CROSSFADE_RATIO: + g_value_set_double (value, pad->crossfade); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); break; @@ -188,6 +193,10 @@ gst_compositor_pad_set_property (GObject * object, guint prop_id, case PROP_PAD_ALPHA: pad->alpha = g_value_get_double (value); break; + case PROP_PAD_CROSSFADE_RATIO: + pad->crossfade = g_value_get_double (value); + GST_VIDEO_AGGREGATOR_PAD (pad)->ABI.needs_alpha = pad->crossfade >= 0.0f; + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); break; @@ -475,11 +484,20 @@ gst_compositor_pad_prepare_frame (GstVideoAggregatorPad * pad, } GST_OBJECT_LOCK (vagg); + /* Check if we are crossfading the pad one way or another */ + l = g_list_find (GST_ELEMENT (vagg)->sinkpads, pad); + if ((l->prev && GST_COMPOSITOR_PAD (l->prev->data)->crossfade >= 0.0) || + (GST_COMPOSITOR_PAD (pad)->crossfade >= 0.0)) { + GST_DEBUG_OBJECT (pad, "Is being crossfaded with previous pad"); + l = NULL; + } else { + l = l->next; + } + /* Check if this frame is obscured by a higher-zorder frame * TODO: Also skip a frame if it's obscured by a combination of * higher-zorder frames */ - for (l = g_list_find (GST_ELEMENT (vagg)->sinkpads, pad)->next; l; - l = l->next) { + for (; l; l = l->next) { GstVideoRectangle frame2_rect; GstVideoAggregatorPad *pad2 = l->data; GstCompositorPad *cpad2 = GST_COMPOSITOR_PAD (pad2); @@ -621,6 +639,12 @@ gst_compositor_pad_class_init (GstCompositorPadClass * klass) g_param_spec_double ("alpha", "Alpha", "Alpha of the picture", 0.0, 1.0, DEFAULT_PAD_ALPHA, G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); + g_object_class_install_property (gobject_class, PROP_PAD_CROSSFADE_RATIO, + g_param_spec_double ("crossfade-ratio", "Crossfade ratio", + "The crossfade ratio to use while crossfading with the following pad." + "A value inferior to 0 means no crossfading.", + -1.0, 1.0, DEFAULT_PAD_CROSSFADE_RATIO, + G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); vaggpadclass->set_info = GST_DEBUG_FUNCPTR (gst_compositor_pad_set_info); vaggpadclass->prepare_frame = @@ -635,6 +659,7 @@ gst_compositor_pad_init (GstCompositorPad * compo_pad) compo_pad->xpos = DEFAULT_PAD_XPOS; compo_pad->ypos = DEFAULT_PAD_YPOS; compo_pad->alpha = DEFAULT_PAD_ALPHA; + compo_pad->crossfade = DEFAULT_PAD_CROSSFADE_RATIO; } @@ -643,7 +668,7 @@ gst_compositor_pad_init (GstCompositorPad * compo_pad) enum { PROP_0, - PROP_BACKGROUND + PROP_BACKGROUND, }; #define GST_TYPE_COMPOSITOR_BACKGROUND (gst_compositor_background_get_type()) @@ -963,6 +988,124 @@ _negotiated_caps (GstAggregator * agg, GstCaps * caps) return GST_AGGREGATOR_CLASS (parent_class)->negotiated_src_caps (agg, caps); } +/* Fills frame with transparent pixels if @nframe is NULL otherwise copy @frame + * properties and fill @nframes with transparent pixels */ +static GstFlowReturn +gst_compositor_fill_transparent (GstCompositor * self, GstVideoFrame * frame, + GstVideoFrame * nframe) +{ + guint plane, num_planes, height, i; + + if (nframe) { + GstBuffer *cbuffer = gst_buffer_copy_deep (frame->buffer); + + if (!gst_video_frame_map (nframe, &frame->info, cbuffer, GST_MAP_WRITE)) { + GST_WARNING_OBJECT (self, "Could not map output buffer"); + return GST_FLOW_ERROR; + } + } else { + nframe = frame; + } + + num_planes = GST_VIDEO_FRAME_N_PLANES (nframe); + for (plane = 0; plane < num_planes; ++plane) { + guint8 *pdata; + gsize rowsize, plane_stride; + + pdata = GST_VIDEO_FRAME_PLANE_DATA (nframe, plane); + plane_stride = GST_VIDEO_FRAME_PLANE_STRIDE (nframe, plane); + rowsize = GST_VIDEO_FRAME_COMP_WIDTH (nframe, plane) + * GST_VIDEO_FRAME_COMP_PSTRIDE (nframe, plane); + height = GST_VIDEO_FRAME_COMP_HEIGHT (nframe, plane); + for (i = 0; i < height; ++i) { + memset (pdata, 0, rowsize); + pdata += plane_stride; + } + } + + return GST_FLOW_OK; +} + +/* WITH GST_OBJECT_LOCK !! + * Returns: %TRUE if outframe is allready ready to be used as we are using + * a transparent background and all pads have already been crossfaded + * %FALSE otherwise + */ +static gboolean +gst_compositor_crossfade_frames (GstCompositor * self, GstVideoFrame * outframe) +{ + GList *l; + gboolean all_crossfading = FALSE; + GstVideoAggregator *vagg = GST_VIDEO_AGGREGATOR (self); + + if (self->background == COMPOSITOR_BACKGROUND_TRANSPARENT) { + + all_crossfading = TRUE; + for (l = GST_ELEMENT (self)->sinkpads; l; l = l->next) { + GstCompositorPad *compo_pad = GST_COMPOSITOR_PAD (l->data); + + if (compo_pad->crossfade < 0.0 && l->next && + GST_COMPOSITOR_PAD (l->next->data)->crossfade < 0) { + all_crossfading = FALSE; + + break; + } + } + } + + for (l = GST_ELEMENT (self)->sinkpads; l; l = l->next) { + GstVideoAggregatorPad *pad = l->data; + GstCompositorPad *compo_pad = GST_COMPOSITOR_PAD (pad); + + if (compo_pad->crossfade >= 0.0f && pad->aggregated_frame) { + gfloat alpha = compo_pad->crossfade * compo_pad->alpha; + GstVideoAggregatorPad *npad = l->next ? l->next->data : NULL; + GstVideoFrame *nframe; + + if (!all_crossfading) { + nframe = g_slice_new0 (GstVideoFrame); + gst_compositor_fill_transparent (self, outframe, nframe); + } else { + nframe = outframe; + } + + self->overlay (pad->aggregated_frame, + compo_pad->crossfaded ? 0 : compo_pad->xpos, + compo_pad->crossfaded ? 0 : compo_pad->ypos, + alpha, nframe, COMPOSITOR_BLEND_MODE_ADDITIVE); + + if (npad && npad->aggregated_frame) { + GstCompositorPad *next_compo_pad = GST_COMPOSITOR_PAD (npad); + + alpha = (1.0 - compo_pad->crossfade) * next_compo_pad->alpha; + self->overlay (npad->aggregated_frame, next_compo_pad->xpos, + next_compo_pad->ypos, alpha, nframe, + COMPOSITOR_BLEND_MODE_ADDITIVE); + + /* Replace frame with current frame */ + gst_compositor_pad_clean_frame (npad, vagg); + npad->aggregated_frame = !all_crossfading ? nframe : NULL; + next_compo_pad->crossfaded = TRUE; + + /* Frame is now consumed, clean it up */ + gst_compositor_pad_clean_frame (pad, vagg); + pad->aggregated_frame = NULL; + } else { + GST_LOG_OBJECT (self, "Simply fading out as no following pad found"); + gst_compositor_pad_clean_frame (pad, vagg); + pad->aggregated_frame = !all_crossfading ? nframe : NULL; + compo_pad->crossfaded = TRUE; + } + } + } + + if (all_crossfading) + for (l = GST_ELEMENT (self)->sinkpads; l; l = l->next) + GST_COMPOSITOR_PAD (l->data)->crossfaded = FALSE; + + return all_crossfading; +} + static GstFlowReturn gst_compositor_aggregate_frames (GstVideoAggregator * vagg, GstBuffer * outbuf) { @@ -992,39 +1135,26 @@ gst_compositor_aggregate_frames (GstVideoAggregator * vagg, GstBuffer * outbuf) self->fill_color (outframe, 240, 128, 128); break; case COMPOSITOR_BACKGROUND_TRANSPARENT: - { - guint i, plane, num_planes, height; - - num_planes = GST_VIDEO_FRAME_N_PLANES (outframe); - for (plane = 0; plane < num_planes; ++plane) { - guint8 *pdata; - gsize rowsize, plane_stride; - - pdata = GST_VIDEO_FRAME_PLANE_DATA (outframe, plane); - plane_stride = GST_VIDEO_FRAME_PLANE_STRIDE (outframe, plane); - rowsize = GST_VIDEO_FRAME_COMP_WIDTH (outframe, plane) - * GST_VIDEO_FRAME_COMP_PSTRIDE (outframe, plane); - height = GST_VIDEO_FRAME_COMP_HEIGHT (outframe, plane); - for (i = 0; i < height; ++i) { - memset (pdata, 0, rowsize); - pdata += plane_stride; - } - } - + gst_compositor_fill_transparent (self, outframe, NULL); /* use overlay to keep background transparent */ composite = self->overlay; break; - } } GST_OBJECT_LOCK (vagg); - for (l = GST_ELEMENT (vagg)->sinkpads; l; l = l->next) { - GstVideoAggregatorPad *pad = l->data; - GstCompositorPad *compo_pad = GST_COMPOSITOR_PAD (pad); + /* First mix the crossfade frames as required */ + if (!gst_compositor_crossfade_frames (self, outframe)) { + for (l = GST_ELEMENT (vagg)->sinkpads; l; l = l->next) { + GstVideoAggregatorPad *pad = l->data; + GstCompositorPad *compo_pad = GST_COMPOSITOR_PAD (pad); - if (pad->aggregated_frame != NULL) { - composite (pad->aggregated_frame, compo_pad->xpos, compo_pad->ypos, - compo_pad->alpha, outframe); + if (pad->aggregated_frame != NULL) { + composite (pad->aggregated_frame, + compo_pad->crossfaded ? 0 : compo_pad->xpos, + compo_pad->crossfaded ? 0 : compo_pad->ypos, compo_pad->alpha, + outframe, COMPOSITOR_BLEND_MODE_NORMAL); + compo_pad->crossfaded = FALSE; + } } } GST_OBJECT_UNLOCK (vagg); @@ -1112,8 +1242,8 @@ gst_compositor_class_init (GstCompositorClass * klass) static void gst_compositor_init (GstCompositor * self) { - self->background = DEFAULT_BACKGROUND; /* initialize variables */ + self->background = DEFAULT_BACKGROUND; } /* Element registration */ diff --git a/gst/compositor/compositor.h b/gst/compositor/compositor.h index a6b4d9fce1..d8cefa3d2a 100644 --- a/gst/compositor/compositor.h +++ b/gst/compositor/compositor.h @@ -17,7 +17,7 @@ * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, * Boston, MA 02110-1301, USA. */ - + #ifndef __GST_COMPOSITOR_H__ #define __GST_COMPOSITOR_H__ @@ -57,8 +57,7 @@ typedef enum COMPOSITOR_BACKGROUND_BLACK, COMPOSITOR_BACKGROUND_WHITE, COMPOSITOR_BACKGROUND_TRANSPARENT, -} -GstCompositorBackground; +} GstCompositorBackground; /** * GstCompositor: diff --git a/gst/compositor/compositororc.orc b/gst/compositor/compositororc.orc index 234ec21128..66c4dcc730 100644 --- a/gst/compositor/compositororc.orc +++ b/gst/compositor/compositororc.orc @@ -98,7 +98,6 @@ x4 convwb t, d_wide orl t, t, a_alpha storel d, t - .function compositor_orc_overlay_argb .flags 2d .dest 4 d guint8 @@ -159,6 +158,76 @@ andl a, a, a_alpha orl t, t, a storel d, t + +.function compositor_orc_overlay_argb_addition +.flags 2d +.dest 4 d guint8 +.source 4 s guint8 +.param 2 alpha +.temp 4 t +.temp 2 tw +.temp 1 tb +.temp 8 alpha_s +.temp 8 alpha_s_inv +.temp 8 alpha_factor +.temp 8 alpha_d +.temp 4 a +.temp 8 d_wide +.temp 8 s_wide +.const 4 xfs 0xffffffff +.const 4 a_alpha 0x000000ff +.const 4 a_alpha_inv 0xffffff00 + +# calc source alpha as alpha_s = alpha_s * alpha / 255 +loadl t, s +convlw tw, t +convwb tb, tw +splatbl a, tb +x4 convubw alpha_s, a +x4 mullw alpha_s, alpha_s, alpha +x4 div255w alpha_s, alpha_s +x4 convubw s_wide, t +x4 mullw s_wide, s_wide, alpha_s + +# calc destination alpha as alpha_factor = (255-alpha_s) * alpha_factor / factor +loadpl a, xfs +x4 convubw alpha_s_inv, a +x4 subw alpha_s_inv, alpha_s_inv, alpha_s +loadl t, d +convlw tw, t +convwb tb, tw +splatbl a, tb +x4 convubw alpha_factor, a +x4 mullw alpha_factor, alpha_factor, alpha_s_inv +x4 div255w alpha_factor, alpha_factor +x4 convubw d_wide, t +x4 mullw d_wide, d_wide, alpha_factor + +# calc final pixel as pix_d = pix_s*alpha_s + pix_d*alpha_factor*(255-alpha_s)/255 +x4 addw d_wide, d_wide, s_wide + +# calc the alpha factor alpha_factor = alpha_s + alpha_factor * (255-alpha_s)/255 +x4 addw alpha_factor, alpha_factor, alpha_s + +# now normalize the pix_d by the final alpha to make it associative +x4 divluw, d_wide, d_wide, alpha_factor + +# calc the final global alpha_d = alpha_d + (alpha_s * (alpha / 255)) +loadl t, d +convlw tw, t +convwb tb, tw +splatbl a, tb +x4 convubw alpha_d, a +x4 addw alpha_d, alpha_d, alpha_s + +# pack the new alpha into the correct spot +x4 convwb t, d_wide +andl t, t, a_alpha_inv +x4 convwb a, alpha_d +andl a, a, a_alpha +orl t, t, a +storel d, t + .function compositor_orc_overlay_bgra .flags 2d .dest 4 d guint8 @@ -221,3 +290,76 @@ x4 convwb a, alpha_d andl a, a, a_alpha orl t, t, a storel d, t + +.function compositor_orc_overlay_bgra_addition +.flags 2d +.dest 4 d guint8 +.source 4 s guint8 +.param 2 alpha +.temp 4 t +.temp 4 t2 +.temp 2 tw +.temp 1 tb +.temp 8 alpha_s +.temp 8 alpha_s_inv +.temp 8 alpha_factor +.temp 8 alpha_d +.temp 4 a +.temp 8 d_wide +.temp 8 s_wide +.const 4 xfs 0xffffffff +.const 4 a_alpha 0xff000000 +.const 4 a_alpha_inv 0x00ffffff + +# calc source alpha as alpha_s = alpha_s * alpha / 255 +loadl t, s +shrul t2, t, 24 +convlw tw, t2 +convwb tb, tw +splatbl a, tb +x4 convubw alpha_s, a +x4 mullw alpha_s, alpha_s, alpha +x4 div255w alpha_s, alpha_s +x4 convubw s_wide, t +x4 mullw s_wide, s_wide, alpha_s + +# calc destination alpha as alpha_factor = (255-alpha_s) * alpha_factor / 255 +loadpl a, xfs +x4 convubw alpha_s_inv, a +x4 subw alpha_s_inv, alpha_s_inv, alpha_s +loadl t, d +shrul t2, t, 24 +convlw tw, t2 +convwb tb, tw +splatbl a, tb +x4 convubw alpha_factor, a +x4 mullw alpha_factor, alpha_factor, alpha_s_inv +x4 div255w alpha_factor, alpha_factor +x4 convubw d_wide, t +x4 mullw d_wide, d_wide, alpha_factor + +# calc final pixel as pix_d = pix_s*alpha_s + pix_d*alpha_factor*(255-alpha_s)/255 +x4 addw d_wide, d_wide, s_wide + +# calc the final destination alpha_factor = alpha_s + alpha_factor * (255-alpha_s)/255 +x4 addw alpha_factor, alpha_factor, alpha_s + +# now normalize the pix_d by the final alpha to make it associative +x4 divluw, d_wide, d_wide, alpha_factor + +# calc the final global alpha_d = alpha_d + (alpha_s * (alpha / 255)) +loadl t, d +shrul t2, t, 24 +convlw tw, t2 +convwb tb, tw +splatbl a, tb +x4 convubw alpha_d, a +x4 addw alpha_d, alpha_d, alpha_s + +# pack the new alpha into the correct spot +x4 convwb t, d_wide +andl t, t, a_alpha_inv +x4 convwb a, alpha_d +andl a, a, a_alpha +orl t, t, a +storel d, t diff --git a/gst/compositor/compositorpad.h b/gst/compositor/compositorpad.h index cb6f7c7356..1c2a271a80 100644 --- a/gst/compositor/compositorpad.h +++ b/gst/compositor/compositorpad.h @@ -52,10 +52,13 @@ struct _GstCompositorPad gint xpos, ypos; gint width, height; gdouble alpha; + gdouble crossfade; GstVideoConverter *convert; GstVideoInfo conversion_info; GstBuffer *converted_buffer; + + gboolean crossfaded; }; struct _GstCompositorPadClass From f90db75ab60d990fae22bf03d304e1c5e4e9ad5d Mon Sep 17 00:00:00 2001 From: Thibault Saunier Date: Tue, 11 Jul 2017 22:04:55 -0400 Subject: [PATCH 308/381] tests: examples: Add a simple crossfade example https://bugzilla.gnome.org/show_bug.cgi?id=784827 --- tests/examples/compositor/crossfade.c | 135 ++++++++++++++++++++++++++ 1 file changed, 135 insertions(+) create mode 100644 tests/examples/compositor/crossfade.c diff --git a/tests/examples/compositor/crossfade.c b/tests/examples/compositor/crossfade.c new file mode 100644 index 0000000000..de8dd45c79 --- /dev/null +++ b/tests/examples/compositor/crossfade.c @@ -0,0 +1,135 @@ +/* + * GStreamer + * Copyright (C) 2017 Thibault Saunier + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +/** + * Simple crossfade example using the compositor element. + * + * Takes a list of uri/path to video files and crossfade them + * for 10 seconds and returns. + */ + +#include +#include +#include +#include + +typedef struct +{ + GstElement *compositor; + guint z_order; + gboolean is_last; +} VideoInfo; + +gchar * +ensure_uri (const gchar * location) +{ + if (gst_uri_is_valid (location)) + return g_strdup (location); + else + return gst_filename_to_uri (location, NULL); +} + +static void +_pad_added_cb (GstElement * decodebin, GstPad * pad, VideoInfo * info) +{ + GstPad *sinkpad = + gst_element_get_request_pad (GST_ELEMENT (info->compositor), "sink_%u"); + + if (!info->is_last) { + GstControlSource *control_source; + + control_source = gst_interpolation_control_source_new (); + + g_object_set (sinkpad, "crossfade-ratio", 1.0, NULL); + gst_object_add_control_binding (GST_OBJECT (sinkpad), + gst_direct_control_binding_new_absolute (GST_OBJECT (sinkpad), + "crossfade-ratio", control_source)); + + g_object_set (control_source, "mode", GST_INTERPOLATION_MODE_LINEAR, NULL); + + gst_timed_value_control_source_set (GST_TIMED_VALUE_CONTROL_SOURCE + (control_source), 0, 1.0); + gst_timed_value_control_source_set (GST_TIMED_VALUE_CONTROL_SOURCE + (control_source), 10 * GST_SECOND, 0.0); + } + g_object_set (sinkpad, "zorder", info->z_order, NULL); + + gst_pad_link (pad, sinkpad); + + g_free (info); +} + +int +main (int argc, char *argv[]) +{ + gint i; + GstMessage *message; + GstElement *compositor, *sink, *pipeline; + GstBus *bus; + + if (argc < 2) { + g_error ("At least 1 valid video file paths/urls need to " "be provided"); + return -1; + } + + gst_init (&argc, &argv); + pipeline = gst_element_factory_make ("pipeline", NULL); + compositor = gst_element_factory_make ("compositor", NULL); + sink = + gst_parse_bin_from_description ("videoconvert ! autovideosink", TRUE, + NULL); + + gst_util_set_object_arg (G_OBJECT (compositor), "background", "black"); + + gst_bin_add_many (GST_BIN (pipeline), compositor, sink, NULL); + g_assert (gst_element_link (compositor, sink)); + + for (i = 1; i < argc; i++) { + gchar *uri = ensure_uri (argv[i]); + VideoInfo *info = g_malloc0 (sizeof (VideoInfo)); + GstElement *uridecodebin = gst_element_factory_make ("uridecodebin", NULL); + + g_object_set (uridecodebin, "uri", uri, "expose-all-streams", FALSE, + "caps", gst_caps_from_string ("video/x-raw(ANY)"), NULL); + + info->compositor = compositor; + info->z_order = i - 1; + info->is_last = (i == (argc - 1)) && (argc > 2); + g_signal_connect (uridecodebin, "pad-added", (GCallback) _pad_added_cb, + info); + + gst_bin_add (GST_BIN (pipeline), uridecodebin); + } + + bus = gst_element_get_bus (pipeline); + gst_element_set_state (pipeline, GST_STATE_PLAYING); + + message = + gst_bus_timed_pop_filtered (bus, 11 * GST_SECOND, + GST_MESSAGE_EOS | GST_MESSAGE_ERROR); + GST_DEBUG_BIN_TO_DOT_FILE_WITH_TS (GST_BIN (pipeline), + GST_DEBUG_GRAPH_SHOW_ALL, "go"); + if (message) + gst_print ("%" GST_PTR_FORMAT "\n", message); + else + gst_print ("Timeout\n"); + gst_element_set_state (pipeline, GST_STATE_NULL); + gst_object_unref (pipeline); +} From 61b54cdb40b68f26eb2cca6e3a03fd8c493b189b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim-Philipp=20M=C3=BCller?= Date: Thu, 3 Aug 2017 20:14:20 +0100 Subject: [PATCH 309/381] compositor: update disted orc fallback files --- gst/compositor/compositororc-dist.c | 1319 +++++++++++++++++++++++++++ gst/compositor/compositororc-dist.h | 2 + 2 files changed, 1321 insertions(+) diff --git a/gst/compositor/compositororc-dist.c b/gst/compositor/compositororc-dist.c index 41744820b6..0fab506cd8 100644 --- a/gst/compositor/compositororc-dist.c +++ b/gst/compositor/compositororc-dist.c @@ -106,8 +106,14 @@ void compositor_orc_blend_bgra (guint8 * ORC_RESTRICT d1, int d1_stride, const guint8 * ORC_RESTRICT s1, int s1_stride, int p1, int n, int m); void compositor_orc_overlay_argb (guint8 * ORC_RESTRICT d1, int d1_stride, const guint8 * ORC_RESTRICT s1, int s1_stride, int p1, int n, int m); +void compositor_orc_overlay_argb_addition (guint8 * ORC_RESTRICT d1, + int d1_stride, const guint8 * ORC_RESTRICT s1, int s1_stride, int p1, int n, + int m); void compositor_orc_overlay_bgra (guint8 * ORC_RESTRICT d1, int d1_stride, const guint8 * ORC_RESTRICT s1, int s1_stride, int p1, int n, int m); +void compositor_orc_overlay_bgra_addition (guint8 * ORC_RESTRICT d1, + int d1_stride, const guint8 * ORC_RESTRICT s1, int s1_stride, int p1, int n, + int m); /* begin Orc C target preamble */ @@ -1881,6 +1887,649 @@ compositor_orc_overlay_argb (guint8 * ORC_RESTRICT d1, int d1_stride, #endif +/* compositor_orc_overlay_argb_addition */ +#ifdef DISABLE_ORC +void +compositor_orc_overlay_argb_addition (guint8 * ORC_RESTRICT d1, int d1_stride, + const guint8 * ORC_RESTRICT s1, int s1_stride, int p1, int n, int m) +{ + int i; + int j; + orc_union32 *ORC_RESTRICT ptr0; + const orc_union32 *ORC_RESTRICT ptr4; + orc_union64 var42; +#if defined(__APPLE__) && __GNUC__ == 4 && __GNUC_MINOR__ == 2 && defined (__i386__) + volatile orc_union32 var43; +#else + orc_union32 var43; +#endif +#if defined(__APPLE__) && __GNUC__ == 4 && __GNUC_MINOR__ == 2 && defined (__i386__) + volatile orc_union32 var44; +#else + orc_union32 var44; +#endif + orc_union32 var45; + orc_union16 var46; + orc_int8 var47; + orc_union32 var48; + orc_union64 var49; + orc_union64 var50; + orc_union64 var51; + orc_union64 var52; + orc_union64 var53; + orc_union32 var54; + orc_union64 var55; + orc_union64 var56; + orc_union32 var57; + orc_union16 var58; + orc_int8 var59; + orc_union32 var60; + orc_union64 var61; + orc_union64 var62; + orc_union64 var63; + orc_union64 var64; + orc_union64 var65; + orc_union64 var66; + orc_union64 var67; + orc_union64 var68; + orc_union32 var69; + orc_union16 var70; + orc_int8 var71; + orc_union32 var72; + orc_union64 var73; + orc_union64 var74; + orc_union32 var75; + orc_union32 var76; + orc_union32 var77; + orc_union32 var78; + orc_union32 var79; + + for (j = 0; j < m; j++) { + ptr0 = ORC_PTR_OFFSET (d1, d1_stride * j); + ptr4 = ORC_PTR_OFFSET (s1, s1_stride * j); + + /* 5: loadpw */ + var42.x4[0] = p1; + var42.x4[1] = p1; + var42.x4[2] = p1; + var42.x4[3] = p1; + /* 10: loadpl */ + var54.i = (int) 0xffffffff; /* -1 or 2.122e-314f */ + /* 32: loadpl */ + var43.i = (int) 0xffffff00; /* -256 or 2.122e-314f */ + /* 35: loadpl */ + var44.i = (int) 0x000000ff; /* 255 or 1.25987e-321f */ + + for (i = 0; i < n; i++) { + /* 0: loadl */ + var45 = ptr4[i]; + /* 1: convlw */ + var46.i = var45.i; + /* 2: convwb */ + var47 = var46.i; + /* 3: splatbl */ + var48.i = + ((((orc_uint32) var47) & 0xff) << 24) | ((((orc_uint32) var47) & 0xff) + << 16) | ((((orc_uint32) var47) & 0xff) << 8) | (((orc_uint32) var47) + & 0xff); + /* 4: convubw */ + var49.x4[0] = (orc_uint8) var48.x4[0]; + var49.x4[1] = (orc_uint8) var48.x4[1]; + var49.x4[2] = (orc_uint8) var48.x4[2]; + var49.x4[3] = (orc_uint8) var48.x4[3]; + /* 6: mullw */ + var50.x4[0] = (var49.x4[0] * var42.x4[0]) & 0xffff; + var50.x4[1] = (var49.x4[1] * var42.x4[1]) & 0xffff; + var50.x4[2] = (var49.x4[2] * var42.x4[2]) & 0xffff; + var50.x4[3] = (var49.x4[3] * var42.x4[3]) & 0xffff; + /* 7: div255w */ + var51.x4[0] = + ((orc_uint16) (((orc_uint16) (var50.x4[0] + 128)) + + (((orc_uint16) (var50.x4[0] + 128)) >> 8))) >> 8; + var51.x4[1] = + ((orc_uint16) (((orc_uint16) (var50.x4[1] + 128)) + + (((orc_uint16) (var50.x4[1] + 128)) >> 8))) >> 8; + var51.x4[2] = + ((orc_uint16) (((orc_uint16) (var50.x4[2] + 128)) + + (((orc_uint16) (var50.x4[2] + 128)) >> 8))) >> 8; + var51.x4[3] = + ((orc_uint16) (((orc_uint16) (var50.x4[3] + 128)) + + (((orc_uint16) (var50.x4[3] + 128)) >> 8))) >> 8; + /* 8: convubw */ + var52.x4[0] = (orc_uint8) var45.x4[0]; + var52.x4[1] = (orc_uint8) var45.x4[1]; + var52.x4[2] = (orc_uint8) var45.x4[2]; + var52.x4[3] = (orc_uint8) var45.x4[3]; + /* 9: mullw */ + var53.x4[0] = (var52.x4[0] * var51.x4[0]) & 0xffff; + var53.x4[1] = (var52.x4[1] * var51.x4[1]) & 0xffff; + var53.x4[2] = (var52.x4[2] * var51.x4[2]) & 0xffff; + var53.x4[3] = (var52.x4[3] * var51.x4[3]) & 0xffff; + /* 11: convubw */ + var55.x4[0] = (orc_uint8) var54.x4[0]; + var55.x4[1] = (orc_uint8) var54.x4[1]; + var55.x4[2] = (orc_uint8) var54.x4[2]; + var55.x4[3] = (orc_uint8) var54.x4[3]; + /* 12: subw */ + var56.x4[0] = var55.x4[0] - var51.x4[0]; + var56.x4[1] = var55.x4[1] - var51.x4[1]; + var56.x4[2] = var55.x4[2] - var51.x4[2]; + var56.x4[3] = var55.x4[3] - var51.x4[3]; + /* 13: loadl */ + var57 = ptr0[i]; + /* 14: convlw */ + var58.i = var57.i; + /* 15: convwb */ + var59 = var58.i; + /* 16: splatbl */ + var60.i = + ((((orc_uint32) var59) & 0xff) << 24) | ((((orc_uint32) var59) & 0xff) + << 16) | ((((orc_uint32) var59) & 0xff) << 8) | (((orc_uint32) var59) + & 0xff); + /* 17: convubw */ + var61.x4[0] = (orc_uint8) var60.x4[0]; + var61.x4[1] = (orc_uint8) var60.x4[1]; + var61.x4[2] = (orc_uint8) var60.x4[2]; + var61.x4[3] = (orc_uint8) var60.x4[3]; + /* 18: mullw */ + var62.x4[0] = (var61.x4[0] * var56.x4[0]) & 0xffff; + var62.x4[1] = (var61.x4[1] * var56.x4[1]) & 0xffff; + var62.x4[2] = (var61.x4[2] * var56.x4[2]) & 0xffff; + var62.x4[3] = (var61.x4[3] * var56.x4[3]) & 0xffff; + /* 19: div255w */ + var63.x4[0] = + ((orc_uint16) (((orc_uint16) (var62.x4[0] + 128)) + + (((orc_uint16) (var62.x4[0] + 128)) >> 8))) >> 8; + var63.x4[1] = + ((orc_uint16) (((orc_uint16) (var62.x4[1] + 128)) + + (((orc_uint16) (var62.x4[1] + 128)) >> 8))) >> 8; + var63.x4[2] = + ((orc_uint16) (((orc_uint16) (var62.x4[2] + 128)) + + (((orc_uint16) (var62.x4[2] + 128)) >> 8))) >> 8; + var63.x4[3] = + ((orc_uint16) (((orc_uint16) (var62.x4[3] + 128)) + + (((orc_uint16) (var62.x4[3] + 128)) >> 8))) >> 8; + /* 20: convubw */ + var64.x4[0] = (orc_uint8) var57.x4[0]; + var64.x4[1] = (orc_uint8) var57.x4[1]; + var64.x4[2] = (orc_uint8) var57.x4[2]; + var64.x4[3] = (orc_uint8) var57.x4[3]; + /* 21: mullw */ + var65.x4[0] = (var64.x4[0] * var63.x4[0]) & 0xffff; + var65.x4[1] = (var64.x4[1] * var63.x4[1]) & 0xffff; + var65.x4[2] = (var64.x4[2] * var63.x4[2]) & 0xffff; + var65.x4[3] = (var64.x4[3] * var63.x4[3]) & 0xffff; + /* 22: addw */ + var66.x4[0] = var65.x4[0] + var53.x4[0]; + var66.x4[1] = var65.x4[1] + var53.x4[1]; + var66.x4[2] = var65.x4[2] + var53.x4[2]; + var66.x4[3] = var65.x4[3] + var53.x4[3]; + /* 23: addw */ + var67.x4[0] = var63.x4[0] + var51.x4[0]; + var67.x4[1] = var63.x4[1] + var51.x4[1]; + var67.x4[2] = var63.x4[2] + var51.x4[2]; + var67.x4[3] = var63.x4[3] + var51.x4[3]; + /* 24: divluw */ + var68.x4[0] = + ((var67.x4[0] & 0xff) == + 0) ? 255 : ORC_CLAMP_UB (((orc_uint16) var66.x4[0]) / + ((orc_uint16) var67.x4[0] & 0xff)); + var68.x4[1] = + ((var67.x4[1] & 0xff) == + 0) ? 255 : ORC_CLAMP_UB (((orc_uint16) var66.x4[1]) / + ((orc_uint16) var67.x4[1] & 0xff)); + var68.x4[2] = + ((var67.x4[2] & 0xff) == + 0) ? 255 : ORC_CLAMP_UB (((orc_uint16) var66.x4[2]) / + ((orc_uint16) var67.x4[2] & 0xff)); + var68.x4[3] = + ((var67.x4[3] & 0xff) == + 0) ? 255 : ORC_CLAMP_UB (((orc_uint16) var66.x4[3]) / + ((orc_uint16) var67.x4[3] & 0xff)); + /* 25: loadl */ + var69 = ptr0[i]; + /* 26: convlw */ + var70.i = var69.i; + /* 27: convwb */ + var71 = var70.i; + /* 28: splatbl */ + var72.i = + ((((orc_uint32) var71) & 0xff) << 24) | ((((orc_uint32) var71) & 0xff) + << 16) | ((((orc_uint32) var71) & 0xff) << 8) | (((orc_uint32) var71) + & 0xff); + /* 29: convubw */ + var73.x4[0] = (orc_uint8) var72.x4[0]; + var73.x4[1] = (orc_uint8) var72.x4[1]; + var73.x4[2] = (orc_uint8) var72.x4[2]; + var73.x4[3] = (orc_uint8) var72.x4[3]; + /* 30: addw */ + var74.x4[0] = var73.x4[0] + var51.x4[0]; + var74.x4[1] = var73.x4[1] + var51.x4[1]; + var74.x4[2] = var73.x4[2] + var51.x4[2]; + var74.x4[3] = var73.x4[3] + var51.x4[3]; + /* 31: convwb */ + var75.x4[0] = var68.x4[0]; + var75.x4[1] = var68.x4[1]; + var75.x4[2] = var68.x4[2]; + var75.x4[3] = var68.x4[3]; + /* 33: andl */ + var76.i = var75.i & var43.i; + /* 34: convwb */ + var77.x4[0] = var74.x4[0]; + var77.x4[1] = var74.x4[1]; + var77.x4[2] = var74.x4[2]; + var77.x4[3] = var74.x4[3]; + /* 36: andl */ + var78.i = var77.i & var44.i; + /* 37: orl */ + var79.i = var76.i | var78.i; + /* 38: storel */ + ptr0[i] = var79; + } + } + +} + +#else +static void +_backup_compositor_orc_overlay_argb_addition (OrcExecutor * ORC_RESTRICT ex) +{ + int i; + int j; + int n = ex->n; + int m = ex->params[ORC_VAR_A1]; + orc_union32 *ORC_RESTRICT ptr0; + const orc_union32 *ORC_RESTRICT ptr4; + orc_union64 var42; +#if defined(__APPLE__) && __GNUC__ == 4 && __GNUC_MINOR__ == 2 && defined (__i386__) + volatile orc_union32 var43; +#else + orc_union32 var43; +#endif +#if defined(__APPLE__) && __GNUC__ == 4 && __GNUC_MINOR__ == 2 && defined (__i386__) + volatile orc_union32 var44; +#else + orc_union32 var44; +#endif + orc_union32 var45; + orc_union16 var46; + orc_int8 var47; + orc_union32 var48; + orc_union64 var49; + orc_union64 var50; + orc_union64 var51; + orc_union64 var52; + orc_union64 var53; + orc_union32 var54; + orc_union64 var55; + orc_union64 var56; + orc_union32 var57; + orc_union16 var58; + orc_int8 var59; + orc_union32 var60; + orc_union64 var61; + orc_union64 var62; + orc_union64 var63; + orc_union64 var64; + orc_union64 var65; + orc_union64 var66; + orc_union64 var67; + orc_union64 var68; + orc_union32 var69; + orc_union16 var70; + orc_int8 var71; + orc_union32 var72; + orc_union64 var73; + orc_union64 var74; + orc_union32 var75; + orc_union32 var76; + orc_union32 var77; + orc_union32 var78; + orc_union32 var79; + + for (j = 0; j < m; j++) { + ptr0 = ORC_PTR_OFFSET (ex->arrays[0], ex->params[0] * j); + ptr4 = ORC_PTR_OFFSET (ex->arrays[4], ex->params[4] * j); + + /* 5: loadpw */ + var42.x4[0] = ex->params[24]; + var42.x4[1] = ex->params[24]; + var42.x4[2] = ex->params[24]; + var42.x4[3] = ex->params[24]; + /* 10: loadpl */ + var54.i = (int) 0xffffffff; /* -1 or 2.122e-314f */ + /* 32: loadpl */ + var43.i = (int) 0xffffff00; /* -256 or 2.122e-314f */ + /* 35: loadpl */ + var44.i = (int) 0x000000ff; /* 255 or 1.25987e-321f */ + + for (i = 0; i < n; i++) { + /* 0: loadl */ + var45 = ptr4[i]; + /* 1: convlw */ + var46.i = var45.i; + /* 2: convwb */ + var47 = var46.i; + /* 3: splatbl */ + var48.i = + ((((orc_uint32) var47) & 0xff) << 24) | ((((orc_uint32) var47) & 0xff) + << 16) | ((((orc_uint32) var47) & 0xff) << 8) | (((orc_uint32) var47) + & 0xff); + /* 4: convubw */ + var49.x4[0] = (orc_uint8) var48.x4[0]; + var49.x4[1] = (orc_uint8) var48.x4[1]; + var49.x4[2] = (orc_uint8) var48.x4[2]; + var49.x4[3] = (orc_uint8) var48.x4[3]; + /* 6: mullw */ + var50.x4[0] = (var49.x4[0] * var42.x4[0]) & 0xffff; + var50.x4[1] = (var49.x4[1] * var42.x4[1]) & 0xffff; + var50.x4[2] = (var49.x4[2] * var42.x4[2]) & 0xffff; + var50.x4[3] = (var49.x4[3] * var42.x4[3]) & 0xffff; + /* 7: div255w */ + var51.x4[0] = + ((orc_uint16) (((orc_uint16) (var50.x4[0] + 128)) + + (((orc_uint16) (var50.x4[0] + 128)) >> 8))) >> 8; + var51.x4[1] = + ((orc_uint16) (((orc_uint16) (var50.x4[1] + 128)) + + (((orc_uint16) (var50.x4[1] + 128)) >> 8))) >> 8; + var51.x4[2] = + ((orc_uint16) (((orc_uint16) (var50.x4[2] + 128)) + + (((orc_uint16) (var50.x4[2] + 128)) >> 8))) >> 8; + var51.x4[3] = + ((orc_uint16) (((orc_uint16) (var50.x4[3] + 128)) + + (((orc_uint16) (var50.x4[3] + 128)) >> 8))) >> 8; + /* 8: convubw */ + var52.x4[0] = (orc_uint8) var45.x4[0]; + var52.x4[1] = (orc_uint8) var45.x4[1]; + var52.x4[2] = (orc_uint8) var45.x4[2]; + var52.x4[3] = (orc_uint8) var45.x4[3]; + /* 9: mullw */ + var53.x4[0] = (var52.x4[0] * var51.x4[0]) & 0xffff; + var53.x4[1] = (var52.x4[1] * var51.x4[1]) & 0xffff; + var53.x4[2] = (var52.x4[2] * var51.x4[2]) & 0xffff; + var53.x4[3] = (var52.x4[3] * var51.x4[3]) & 0xffff; + /* 11: convubw */ + var55.x4[0] = (orc_uint8) var54.x4[0]; + var55.x4[1] = (orc_uint8) var54.x4[1]; + var55.x4[2] = (orc_uint8) var54.x4[2]; + var55.x4[3] = (orc_uint8) var54.x4[3]; + /* 12: subw */ + var56.x4[0] = var55.x4[0] - var51.x4[0]; + var56.x4[1] = var55.x4[1] - var51.x4[1]; + var56.x4[2] = var55.x4[2] - var51.x4[2]; + var56.x4[3] = var55.x4[3] - var51.x4[3]; + /* 13: loadl */ + var57 = ptr0[i]; + /* 14: convlw */ + var58.i = var57.i; + /* 15: convwb */ + var59 = var58.i; + /* 16: splatbl */ + var60.i = + ((((orc_uint32) var59) & 0xff) << 24) | ((((orc_uint32) var59) & 0xff) + << 16) | ((((orc_uint32) var59) & 0xff) << 8) | (((orc_uint32) var59) + & 0xff); + /* 17: convubw */ + var61.x4[0] = (orc_uint8) var60.x4[0]; + var61.x4[1] = (orc_uint8) var60.x4[1]; + var61.x4[2] = (orc_uint8) var60.x4[2]; + var61.x4[3] = (orc_uint8) var60.x4[3]; + /* 18: mullw */ + var62.x4[0] = (var61.x4[0] * var56.x4[0]) & 0xffff; + var62.x4[1] = (var61.x4[1] * var56.x4[1]) & 0xffff; + var62.x4[2] = (var61.x4[2] * var56.x4[2]) & 0xffff; + var62.x4[3] = (var61.x4[3] * var56.x4[3]) & 0xffff; + /* 19: div255w */ + var63.x4[0] = + ((orc_uint16) (((orc_uint16) (var62.x4[0] + 128)) + + (((orc_uint16) (var62.x4[0] + 128)) >> 8))) >> 8; + var63.x4[1] = + ((orc_uint16) (((orc_uint16) (var62.x4[1] + 128)) + + (((orc_uint16) (var62.x4[1] + 128)) >> 8))) >> 8; + var63.x4[2] = + ((orc_uint16) (((orc_uint16) (var62.x4[2] + 128)) + + (((orc_uint16) (var62.x4[2] + 128)) >> 8))) >> 8; + var63.x4[3] = + ((orc_uint16) (((orc_uint16) (var62.x4[3] + 128)) + + (((orc_uint16) (var62.x4[3] + 128)) >> 8))) >> 8; + /* 20: convubw */ + var64.x4[0] = (orc_uint8) var57.x4[0]; + var64.x4[1] = (orc_uint8) var57.x4[1]; + var64.x4[2] = (orc_uint8) var57.x4[2]; + var64.x4[3] = (orc_uint8) var57.x4[3]; + /* 21: mullw */ + var65.x4[0] = (var64.x4[0] * var63.x4[0]) & 0xffff; + var65.x4[1] = (var64.x4[1] * var63.x4[1]) & 0xffff; + var65.x4[2] = (var64.x4[2] * var63.x4[2]) & 0xffff; + var65.x4[3] = (var64.x4[3] * var63.x4[3]) & 0xffff; + /* 22: addw */ + var66.x4[0] = var65.x4[0] + var53.x4[0]; + var66.x4[1] = var65.x4[1] + var53.x4[1]; + var66.x4[2] = var65.x4[2] + var53.x4[2]; + var66.x4[3] = var65.x4[3] + var53.x4[3]; + /* 23: addw */ + var67.x4[0] = var63.x4[0] + var51.x4[0]; + var67.x4[1] = var63.x4[1] + var51.x4[1]; + var67.x4[2] = var63.x4[2] + var51.x4[2]; + var67.x4[3] = var63.x4[3] + var51.x4[3]; + /* 24: divluw */ + var68.x4[0] = + ((var67.x4[0] & 0xff) == + 0) ? 255 : ORC_CLAMP_UB (((orc_uint16) var66.x4[0]) / + ((orc_uint16) var67.x4[0] & 0xff)); + var68.x4[1] = + ((var67.x4[1] & 0xff) == + 0) ? 255 : ORC_CLAMP_UB (((orc_uint16) var66.x4[1]) / + ((orc_uint16) var67.x4[1] & 0xff)); + var68.x4[2] = + ((var67.x4[2] & 0xff) == + 0) ? 255 : ORC_CLAMP_UB (((orc_uint16) var66.x4[2]) / + ((orc_uint16) var67.x4[2] & 0xff)); + var68.x4[3] = + ((var67.x4[3] & 0xff) == + 0) ? 255 : ORC_CLAMP_UB (((orc_uint16) var66.x4[3]) / + ((orc_uint16) var67.x4[3] & 0xff)); + /* 25: loadl */ + var69 = ptr0[i]; + /* 26: convlw */ + var70.i = var69.i; + /* 27: convwb */ + var71 = var70.i; + /* 28: splatbl */ + var72.i = + ((((orc_uint32) var71) & 0xff) << 24) | ((((orc_uint32) var71) & 0xff) + << 16) | ((((orc_uint32) var71) & 0xff) << 8) | (((orc_uint32) var71) + & 0xff); + /* 29: convubw */ + var73.x4[0] = (orc_uint8) var72.x4[0]; + var73.x4[1] = (orc_uint8) var72.x4[1]; + var73.x4[2] = (orc_uint8) var72.x4[2]; + var73.x4[3] = (orc_uint8) var72.x4[3]; + /* 30: addw */ + var74.x4[0] = var73.x4[0] + var51.x4[0]; + var74.x4[1] = var73.x4[1] + var51.x4[1]; + var74.x4[2] = var73.x4[2] + var51.x4[2]; + var74.x4[3] = var73.x4[3] + var51.x4[3]; + /* 31: convwb */ + var75.x4[0] = var68.x4[0]; + var75.x4[1] = var68.x4[1]; + var75.x4[2] = var68.x4[2]; + var75.x4[3] = var68.x4[3]; + /* 33: andl */ + var76.i = var75.i & var43.i; + /* 34: convwb */ + var77.x4[0] = var74.x4[0]; + var77.x4[1] = var74.x4[1]; + var77.x4[2] = var74.x4[2]; + var77.x4[3] = var74.x4[3]; + /* 36: andl */ + var78.i = var77.i & var44.i; + /* 37: orl */ + var79.i = var76.i | var78.i; + /* 38: storel */ + ptr0[i] = var79; + } + } + +} + +void +compositor_orc_overlay_argb_addition (guint8 * ORC_RESTRICT d1, int d1_stride, + const guint8 * ORC_RESTRICT s1, int s1_stride, int p1, int n, int m) +{ + OrcExecutor _ex, *ex = &_ex; + static volatile int p_inited = 0; + static OrcCode *c = 0; + void (*func) (OrcExecutor *); + + if (!p_inited) { + orc_once_mutex_lock (); + if (!p_inited) { + OrcProgram *p; + +#if 1 + static const orc_uint8 bc[] = { + 1, 7, 9, 36, 99, 111, 109, 112, 111, 115, 105, 116, 111, 114, 95, 111, + 114, 99, 95, 111, 118, 101, 114, 108, 97, 121, 95, 97, 114, 103, 98, 95, + 97, 100, 100, 105, 116, 105, 111, 110, 11, 4, 4, 12, 4, 4, 14, 4, + 255, 255, 255, 255, 14, 4, 255, 0, 0, 0, 14, 4, 0, 255, 255, 255, + 16, 2, 20, 4, 20, 2, 20, 1, 20, 8, 20, 8, 20, 8, 20, 8, + 20, 4, 20, 8, 20, 8, 113, 32, 4, 163, 33, 32, 157, 34, 33, 152, + 39, 34, 21, 2, 150, 35, 39, 21, 2, 89, 35, 35, 24, 21, 2, 80, + 35, 35, 21, 2, 150, 41, 32, 21, 2, 89, 41, 41, 35, 115, 39, 16, + 21, 2, 150, 36, 39, 21, 2, 98, 36, 36, 35, 113, 32, 0, 163, 33, + 32, 157, 34, 33, 152, 39, 34, 21, 2, 150, 37, 39, 21, 2, 89, 37, + 37, 36, 21, 2, 80, 37, 37, 21, 2, 150, 40, 32, 21, 2, 89, 40, + 40, 37, 21, 2, 70, 40, 40, 41, 21, 2, 70, 37, 37, 35, 21, 2, + 81, 40, 40, 37, 113, 32, 0, 163, 33, 32, 157, 34, 33, 152, 39, 34, + 21, 2, 150, 38, 39, 21, 2, 70, 38, 38, 35, 21, 2, 157, 32, 40, + 106, 32, 32, 18, 21, 2, 157, 39, 38, 106, 39, 39, 17, 123, 32, 32, + 39, 128, 0, 32, 2, 0, + }; + p = orc_program_new_from_static_bytecode (bc); + orc_program_set_backup_function (p, + _backup_compositor_orc_overlay_argb_addition); +#else + p = orc_program_new (); + orc_program_set_2d (p); + orc_program_set_name (p, "compositor_orc_overlay_argb_addition"); + orc_program_set_backup_function (p, + _backup_compositor_orc_overlay_argb_addition); + orc_program_add_destination (p, 4, "d1"); + orc_program_add_source (p, 4, "s1"); + orc_program_add_constant (p, 4, 0xffffffff, "c1"); + orc_program_add_constant (p, 4, 0x000000ff, "c2"); + orc_program_add_constant (p, 4, 0xffffff00, "c3"); + orc_program_add_parameter (p, 2, "p1"); + orc_program_add_temporary (p, 4, "t1"); + orc_program_add_temporary (p, 2, "t2"); + orc_program_add_temporary (p, 1, "t3"); + orc_program_add_temporary (p, 8, "t4"); + orc_program_add_temporary (p, 8, "t5"); + orc_program_add_temporary (p, 8, "t6"); + orc_program_add_temporary (p, 8, "t7"); + orc_program_add_temporary (p, 4, "t8"); + orc_program_add_temporary (p, 8, "t9"); + orc_program_add_temporary (p, 8, "t10"); + + orc_program_append_2 (p, "loadl", 0, ORC_VAR_T1, ORC_VAR_S1, ORC_VAR_D1, + ORC_VAR_D1); + orc_program_append_2 (p, "convlw", 0, ORC_VAR_T2, ORC_VAR_T1, ORC_VAR_D1, + ORC_VAR_D1); + orc_program_append_2 (p, "convwb", 0, ORC_VAR_T3, ORC_VAR_T2, ORC_VAR_D1, + ORC_VAR_D1); + orc_program_append_2 (p, "splatbl", 0, ORC_VAR_T8, ORC_VAR_T3, ORC_VAR_D1, + ORC_VAR_D1); + orc_program_append_2 (p, "convubw", 2, ORC_VAR_T4, ORC_VAR_T8, ORC_VAR_D1, + ORC_VAR_D1); + orc_program_append_2 (p, "mullw", 2, ORC_VAR_T4, ORC_VAR_T4, ORC_VAR_P1, + ORC_VAR_D1); + orc_program_append_2 (p, "div255w", 2, ORC_VAR_T4, ORC_VAR_T4, ORC_VAR_D1, + ORC_VAR_D1); + orc_program_append_2 (p, "convubw", 2, ORC_VAR_T10, ORC_VAR_T1, + ORC_VAR_D1, ORC_VAR_D1); + orc_program_append_2 (p, "mullw", 2, ORC_VAR_T10, ORC_VAR_T10, ORC_VAR_T4, + ORC_VAR_D1); + orc_program_append_2 (p, "loadpl", 0, ORC_VAR_T8, ORC_VAR_C1, ORC_VAR_D1, + ORC_VAR_D1); + orc_program_append_2 (p, "convubw", 2, ORC_VAR_T5, ORC_VAR_T8, ORC_VAR_D1, + ORC_VAR_D1); + orc_program_append_2 (p, "subw", 2, ORC_VAR_T5, ORC_VAR_T5, ORC_VAR_T4, + ORC_VAR_D1); + orc_program_append_2 (p, "loadl", 0, ORC_VAR_T1, ORC_VAR_D1, ORC_VAR_D1, + ORC_VAR_D1); + orc_program_append_2 (p, "convlw", 0, ORC_VAR_T2, ORC_VAR_T1, ORC_VAR_D1, + ORC_VAR_D1); + orc_program_append_2 (p, "convwb", 0, ORC_VAR_T3, ORC_VAR_T2, ORC_VAR_D1, + ORC_VAR_D1); + orc_program_append_2 (p, "splatbl", 0, ORC_VAR_T8, ORC_VAR_T3, ORC_VAR_D1, + ORC_VAR_D1); + orc_program_append_2 (p, "convubw", 2, ORC_VAR_T6, ORC_VAR_T8, ORC_VAR_D1, + ORC_VAR_D1); + orc_program_append_2 (p, "mullw", 2, ORC_VAR_T6, ORC_VAR_T6, ORC_VAR_T5, + ORC_VAR_D1); + orc_program_append_2 (p, "div255w", 2, ORC_VAR_T6, ORC_VAR_T6, ORC_VAR_D1, + ORC_VAR_D1); + orc_program_append_2 (p, "convubw", 2, ORC_VAR_T9, ORC_VAR_T1, ORC_VAR_D1, + ORC_VAR_D1); + orc_program_append_2 (p, "mullw", 2, ORC_VAR_T9, ORC_VAR_T9, ORC_VAR_T6, + ORC_VAR_D1); + orc_program_append_2 (p, "addw", 2, ORC_VAR_T9, ORC_VAR_T9, ORC_VAR_T10, + ORC_VAR_D1); + orc_program_append_2 (p, "addw", 2, ORC_VAR_T6, ORC_VAR_T6, ORC_VAR_T4, + ORC_VAR_D1); + orc_program_append_2 (p, "divluw", 2, ORC_VAR_T9, ORC_VAR_T9, ORC_VAR_T6, + ORC_VAR_D1); + orc_program_append_2 (p, "loadl", 0, ORC_VAR_T1, ORC_VAR_D1, ORC_VAR_D1, + ORC_VAR_D1); + orc_program_append_2 (p, "convlw", 0, ORC_VAR_T2, ORC_VAR_T1, ORC_VAR_D1, + ORC_VAR_D1); + orc_program_append_2 (p, "convwb", 0, ORC_VAR_T3, ORC_VAR_T2, ORC_VAR_D1, + ORC_VAR_D1); + orc_program_append_2 (p, "splatbl", 0, ORC_VAR_T8, ORC_VAR_T3, ORC_VAR_D1, + ORC_VAR_D1); + orc_program_append_2 (p, "convubw", 2, ORC_VAR_T7, ORC_VAR_T8, ORC_VAR_D1, + ORC_VAR_D1); + orc_program_append_2 (p, "addw", 2, ORC_VAR_T7, ORC_VAR_T7, ORC_VAR_T4, + ORC_VAR_D1); + orc_program_append_2 (p, "convwb", 2, ORC_VAR_T1, ORC_VAR_T9, ORC_VAR_D1, + ORC_VAR_D1); + orc_program_append_2 (p, "andl", 0, ORC_VAR_T1, ORC_VAR_T1, ORC_VAR_C3, + ORC_VAR_D1); + orc_program_append_2 (p, "convwb", 2, ORC_VAR_T8, ORC_VAR_T7, ORC_VAR_D1, + ORC_VAR_D1); + orc_program_append_2 (p, "andl", 0, ORC_VAR_T8, ORC_VAR_T8, ORC_VAR_C2, + ORC_VAR_D1); + orc_program_append_2 (p, "orl", 0, ORC_VAR_T1, ORC_VAR_T1, ORC_VAR_T8, + ORC_VAR_D1); + orc_program_append_2 (p, "storel", 0, ORC_VAR_D1, ORC_VAR_T1, ORC_VAR_D1, + ORC_VAR_D1); +#endif + + orc_program_compile (p); + c = orc_program_take_code (p); + orc_program_free (p); + } + p_inited = TRUE; + orc_once_mutex_unlock (); + } + ex->arrays[ORC_VAR_A2] = c; + ex->program = 0; + + ex->n = n; + ORC_EXECUTOR_M (ex) = m; + ex->arrays[ORC_VAR_D1] = d1; + ex->params[ORC_VAR_D1] = d1_stride; + ex->arrays[ORC_VAR_S1] = (void *) s1; + ex->params[ORC_VAR_S1] = s1_stride; + ex->params[ORC_VAR_P1] = p1; + + func = c->exec; + func (ex); +} +#endif + + /* compositor_orc_overlay_bgra */ #ifdef DISABLE_ORC void @@ -2470,3 +3119,673 @@ compositor_orc_overlay_bgra (guint8 * ORC_RESTRICT d1, int d1_stride, func (ex); } #endif + + +/* compositor_orc_overlay_bgra_addition */ +#ifdef DISABLE_ORC +void +compositor_orc_overlay_bgra_addition (guint8 * ORC_RESTRICT d1, int d1_stride, + const guint8 * ORC_RESTRICT s1, int s1_stride, int p1, int n, int m) +{ + int i; + int j; + orc_union32 *ORC_RESTRICT ptr0; + const orc_union32 *ORC_RESTRICT ptr4; + orc_union64 var43; +#if defined(__APPLE__) && __GNUC__ == 4 && __GNUC_MINOR__ == 2 && defined (__i386__) + volatile orc_union32 var44; +#else + orc_union32 var44; +#endif +#if defined(__APPLE__) && __GNUC__ == 4 && __GNUC_MINOR__ == 2 && defined (__i386__) + volatile orc_union32 var45; +#else + orc_union32 var45; +#endif + orc_union32 var46; + orc_union32 var47; + orc_union16 var48; + orc_int8 var49; + orc_union32 var50; + orc_union64 var51; + orc_union64 var52; + orc_union64 var53; + orc_union64 var54; + orc_union64 var55; + orc_union32 var56; + orc_union64 var57; + orc_union64 var58; + orc_union32 var59; + orc_union32 var60; + orc_union16 var61; + orc_int8 var62; + orc_union32 var63; + orc_union64 var64; + orc_union64 var65; + orc_union64 var66; + orc_union64 var67; + orc_union64 var68; + orc_union64 var69; + orc_union64 var70; + orc_union64 var71; + orc_union32 var72; + orc_union32 var73; + orc_union16 var74; + orc_int8 var75; + orc_union32 var76; + orc_union64 var77; + orc_union64 var78; + orc_union32 var79; + orc_union32 var80; + orc_union32 var81; + orc_union32 var82; + orc_union32 var83; + + for (j = 0; j < m; j++) { + ptr0 = ORC_PTR_OFFSET (d1, d1_stride * j); + ptr4 = ORC_PTR_OFFSET (s1, s1_stride * j); + + /* 6: loadpw */ + var43.x4[0] = p1; + var43.x4[1] = p1; + var43.x4[2] = p1; + var43.x4[3] = p1; + /* 11: loadpl */ + var56.i = (int) 0xffffffff; /* -1 or 2.122e-314f */ + /* 35: loadpl */ + var44.i = (int) 0x00ffffff; /* 16777215 or 8.28905e-317f */ + /* 38: loadpl */ + var45.i = (int) 0xff000000; /* -16777216 or 2.11371e-314f */ + + for (i = 0; i < n; i++) { + /* 0: loadl */ + var46 = ptr4[i]; + /* 1: shrul */ + var47.i = ((orc_uint32) var46.i) >> 24; + /* 2: convlw */ + var48.i = var47.i; + /* 3: convwb */ + var49 = var48.i; + /* 4: splatbl */ + var50.i = + ((((orc_uint32) var49) & 0xff) << 24) | ((((orc_uint32) var49) & 0xff) + << 16) | ((((orc_uint32) var49) & 0xff) << 8) | (((orc_uint32) var49) + & 0xff); + /* 5: convubw */ + var51.x4[0] = (orc_uint8) var50.x4[0]; + var51.x4[1] = (orc_uint8) var50.x4[1]; + var51.x4[2] = (orc_uint8) var50.x4[2]; + var51.x4[3] = (orc_uint8) var50.x4[3]; + /* 7: mullw */ + var52.x4[0] = (var51.x4[0] * var43.x4[0]) & 0xffff; + var52.x4[1] = (var51.x4[1] * var43.x4[1]) & 0xffff; + var52.x4[2] = (var51.x4[2] * var43.x4[2]) & 0xffff; + var52.x4[3] = (var51.x4[3] * var43.x4[3]) & 0xffff; + /* 8: div255w */ + var53.x4[0] = + ((orc_uint16) (((orc_uint16) (var52.x4[0] + 128)) + + (((orc_uint16) (var52.x4[0] + 128)) >> 8))) >> 8; + var53.x4[1] = + ((orc_uint16) (((orc_uint16) (var52.x4[1] + 128)) + + (((orc_uint16) (var52.x4[1] + 128)) >> 8))) >> 8; + var53.x4[2] = + ((orc_uint16) (((orc_uint16) (var52.x4[2] + 128)) + + (((orc_uint16) (var52.x4[2] + 128)) >> 8))) >> 8; + var53.x4[3] = + ((orc_uint16) (((orc_uint16) (var52.x4[3] + 128)) + + (((orc_uint16) (var52.x4[3] + 128)) >> 8))) >> 8; + /* 9: convubw */ + var54.x4[0] = (orc_uint8) var46.x4[0]; + var54.x4[1] = (orc_uint8) var46.x4[1]; + var54.x4[2] = (orc_uint8) var46.x4[2]; + var54.x4[3] = (orc_uint8) var46.x4[3]; + /* 10: mullw */ + var55.x4[0] = (var54.x4[0] * var53.x4[0]) & 0xffff; + var55.x4[1] = (var54.x4[1] * var53.x4[1]) & 0xffff; + var55.x4[2] = (var54.x4[2] * var53.x4[2]) & 0xffff; + var55.x4[3] = (var54.x4[3] * var53.x4[3]) & 0xffff; + /* 12: convubw */ + var57.x4[0] = (orc_uint8) var56.x4[0]; + var57.x4[1] = (orc_uint8) var56.x4[1]; + var57.x4[2] = (orc_uint8) var56.x4[2]; + var57.x4[3] = (orc_uint8) var56.x4[3]; + /* 13: subw */ + var58.x4[0] = var57.x4[0] - var53.x4[0]; + var58.x4[1] = var57.x4[1] - var53.x4[1]; + var58.x4[2] = var57.x4[2] - var53.x4[2]; + var58.x4[3] = var57.x4[3] - var53.x4[3]; + /* 14: loadl */ + var59 = ptr0[i]; + /* 15: shrul */ + var60.i = ((orc_uint32) var59.i) >> 24; + /* 16: convlw */ + var61.i = var60.i; + /* 17: convwb */ + var62 = var61.i; + /* 18: splatbl */ + var63.i = + ((((orc_uint32) var62) & 0xff) << 24) | ((((orc_uint32) var62) & 0xff) + << 16) | ((((orc_uint32) var62) & 0xff) << 8) | (((orc_uint32) var62) + & 0xff); + /* 19: convubw */ + var64.x4[0] = (orc_uint8) var63.x4[0]; + var64.x4[1] = (orc_uint8) var63.x4[1]; + var64.x4[2] = (orc_uint8) var63.x4[2]; + var64.x4[3] = (orc_uint8) var63.x4[3]; + /* 20: mullw */ + var65.x4[0] = (var64.x4[0] * var58.x4[0]) & 0xffff; + var65.x4[1] = (var64.x4[1] * var58.x4[1]) & 0xffff; + var65.x4[2] = (var64.x4[2] * var58.x4[2]) & 0xffff; + var65.x4[3] = (var64.x4[3] * var58.x4[3]) & 0xffff; + /* 21: div255w */ + var66.x4[0] = + ((orc_uint16) (((orc_uint16) (var65.x4[0] + 128)) + + (((orc_uint16) (var65.x4[0] + 128)) >> 8))) >> 8; + var66.x4[1] = + ((orc_uint16) (((orc_uint16) (var65.x4[1] + 128)) + + (((orc_uint16) (var65.x4[1] + 128)) >> 8))) >> 8; + var66.x4[2] = + ((orc_uint16) (((orc_uint16) (var65.x4[2] + 128)) + + (((orc_uint16) (var65.x4[2] + 128)) >> 8))) >> 8; + var66.x4[3] = + ((orc_uint16) (((orc_uint16) (var65.x4[3] + 128)) + + (((orc_uint16) (var65.x4[3] + 128)) >> 8))) >> 8; + /* 22: convubw */ + var67.x4[0] = (orc_uint8) var59.x4[0]; + var67.x4[1] = (orc_uint8) var59.x4[1]; + var67.x4[2] = (orc_uint8) var59.x4[2]; + var67.x4[3] = (orc_uint8) var59.x4[3]; + /* 23: mullw */ + var68.x4[0] = (var67.x4[0] * var66.x4[0]) & 0xffff; + var68.x4[1] = (var67.x4[1] * var66.x4[1]) & 0xffff; + var68.x4[2] = (var67.x4[2] * var66.x4[2]) & 0xffff; + var68.x4[3] = (var67.x4[3] * var66.x4[3]) & 0xffff; + /* 24: addw */ + var69.x4[0] = var68.x4[0] + var55.x4[0]; + var69.x4[1] = var68.x4[1] + var55.x4[1]; + var69.x4[2] = var68.x4[2] + var55.x4[2]; + var69.x4[3] = var68.x4[3] + var55.x4[3]; + /* 25: addw */ + var70.x4[0] = var66.x4[0] + var53.x4[0]; + var70.x4[1] = var66.x4[1] + var53.x4[1]; + var70.x4[2] = var66.x4[2] + var53.x4[2]; + var70.x4[3] = var66.x4[3] + var53.x4[3]; + /* 26: divluw */ + var71.x4[0] = + ((var70.x4[0] & 0xff) == + 0) ? 255 : ORC_CLAMP_UB (((orc_uint16) var69.x4[0]) / + ((orc_uint16) var70.x4[0] & 0xff)); + var71.x4[1] = + ((var70.x4[1] & 0xff) == + 0) ? 255 : ORC_CLAMP_UB (((orc_uint16) var69.x4[1]) / + ((orc_uint16) var70.x4[1] & 0xff)); + var71.x4[2] = + ((var70.x4[2] & 0xff) == + 0) ? 255 : ORC_CLAMP_UB (((orc_uint16) var69.x4[2]) / + ((orc_uint16) var70.x4[2] & 0xff)); + var71.x4[3] = + ((var70.x4[3] & 0xff) == + 0) ? 255 : ORC_CLAMP_UB (((orc_uint16) var69.x4[3]) / + ((orc_uint16) var70.x4[3] & 0xff)); + /* 27: loadl */ + var72 = ptr0[i]; + /* 28: shrul */ + var73.i = ((orc_uint32) var72.i) >> 24; + /* 29: convlw */ + var74.i = var73.i; + /* 30: convwb */ + var75 = var74.i; + /* 31: splatbl */ + var76.i = + ((((orc_uint32) var75) & 0xff) << 24) | ((((orc_uint32) var75) & 0xff) + << 16) | ((((orc_uint32) var75) & 0xff) << 8) | (((orc_uint32) var75) + & 0xff); + /* 32: convubw */ + var77.x4[0] = (orc_uint8) var76.x4[0]; + var77.x4[1] = (orc_uint8) var76.x4[1]; + var77.x4[2] = (orc_uint8) var76.x4[2]; + var77.x4[3] = (orc_uint8) var76.x4[3]; + /* 33: addw */ + var78.x4[0] = var77.x4[0] + var53.x4[0]; + var78.x4[1] = var77.x4[1] + var53.x4[1]; + var78.x4[2] = var77.x4[2] + var53.x4[2]; + var78.x4[3] = var77.x4[3] + var53.x4[3]; + /* 34: convwb */ + var79.x4[0] = var71.x4[0]; + var79.x4[1] = var71.x4[1]; + var79.x4[2] = var71.x4[2]; + var79.x4[3] = var71.x4[3]; + /* 36: andl */ + var80.i = var79.i & var44.i; + /* 37: convwb */ + var81.x4[0] = var78.x4[0]; + var81.x4[1] = var78.x4[1]; + var81.x4[2] = var78.x4[2]; + var81.x4[3] = var78.x4[3]; + /* 39: andl */ + var82.i = var81.i & var45.i; + /* 40: orl */ + var83.i = var80.i | var82.i; + /* 41: storel */ + ptr0[i] = var83; + } + } + +} + +#else +static void +_backup_compositor_orc_overlay_bgra_addition (OrcExecutor * ORC_RESTRICT ex) +{ + int i; + int j; + int n = ex->n; + int m = ex->params[ORC_VAR_A1]; + orc_union32 *ORC_RESTRICT ptr0; + const orc_union32 *ORC_RESTRICT ptr4; + orc_union64 var43; +#if defined(__APPLE__) && __GNUC__ == 4 && __GNUC_MINOR__ == 2 && defined (__i386__) + volatile orc_union32 var44; +#else + orc_union32 var44; +#endif +#if defined(__APPLE__) && __GNUC__ == 4 && __GNUC_MINOR__ == 2 && defined (__i386__) + volatile orc_union32 var45; +#else + orc_union32 var45; +#endif + orc_union32 var46; + orc_union32 var47; + orc_union16 var48; + orc_int8 var49; + orc_union32 var50; + orc_union64 var51; + orc_union64 var52; + orc_union64 var53; + orc_union64 var54; + orc_union64 var55; + orc_union32 var56; + orc_union64 var57; + orc_union64 var58; + orc_union32 var59; + orc_union32 var60; + orc_union16 var61; + orc_int8 var62; + orc_union32 var63; + orc_union64 var64; + orc_union64 var65; + orc_union64 var66; + orc_union64 var67; + orc_union64 var68; + orc_union64 var69; + orc_union64 var70; + orc_union64 var71; + orc_union32 var72; + orc_union32 var73; + orc_union16 var74; + orc_int8 var75; + orc_union32 var76; + orc_union64 var77; + orc_union64 var78; + orc_union32 var79; + orc_union32 var80; + orc_union32 var81; + orc_union32 var82; + orc_union32 var83; + + for (j = 0; j < m; j++) { + ptr0 = ORC_PTR_OFFSET (ex->arrays[0], ex->params[0] * j); + ptr4 = ORC_PTR_OFFSET (ex->arrays[4], ex->params[4] * j); + + /* 6: loadpw */ + var43.x4[0] = ex->params[24]; + var43.x4[1] = ex->params[24]; + var43.x4[2] = ex->params[24]; + var43.x4[3] = ex->params[24]; + /* 11: loadpl */ + var56.i = (int) 0xffffffff; /* -1 or 2.122e-314f */ + /* 35: loadpl */ + var44.i = (int) 0x00ffffff; /* 16777215 or 8.28905e-317f */ + /* 38: loadpl */ + var45.i = (int) 0xff000000; /* -16777216 or 2.11371e-314f */ + + for (i = 0; i < n; i++) { + /* 0: loadl */ + var46 = ptr4[i]; + /* 1: shrul */ + var47.i = ((orc_uint32) var46.i) >> 24; + /* 2: convlw */ + var48.i = var47.i; + /* 3: convwb */ + var49 = var48.i; + /* 4: splatbl */ + var50.i = + ((((orc_uint32) var49) & 0xff) << 24) | ((((orc_uint32) var49) & 0xff) + << 16) | ((((orc_uint32) var49) & 0xff) << 8) | (((orc_uint32) var49) + & 0xff); + /* 5: convubw */ + var51.x4[0] = (orc_uint8) var50.x4[0]; + var51.x4[1] = (orc_uint8) var50.x4[1]; + var51.x4[2] = (orc_uint8) var50.x4[2]; + var51.x4[3] = (orc_uint8) var50.x4[3]; + /* 7: mullw */ + var52.x4[0] = (var51.x4[0] * var43.x4[0]) & 0xffff; + var52.x4[1] = (var51.x4[1] * var43.x4[1]) & 0xffff; + var52.x4[2] = (var51.x4[2] * var43.x4[2]) & 0xffff; + var52.x4[3] = (var51.x4[3] * var43.x4[3]) & 0xffff; + /* 8: div255w */ + var53.x4[0] = + ((orc_uint16) (((orc_uint16) (var52.x4[0] + 128)) + + (((orc_uint16) (var52.x4[0] + 128)) >> 8))) >> 8; + var53.x4[1] = + ((orc_uint16) (((orc_uint16) (var52.x4[1] + 128)) + + (((orc_uint16) (var52.x4[1] + 128)) >> 8))) >> 8; + var53.x4[2] = + ((orc_uint16) (((orc_uint16) (var52.x4[2] + 128)) + + (((orc_uint16) (var52.x4[2] + 128)) >> 8))) >> 8; + var53.x4[3] = + ((orc_uint16) (((orc_uint16) (var52.x4[3] + 128)) + + (((orc_uint16) (var52.x4[3] + 128)) >> 8))) >> 8; + /* 9: convubw */ + var54.x4[0] = (orc_uint8) var46.x4[0]; + var54.x4[1] = (orc_uint8) var46.x4[1]; + var54.x4[2] = (orc_uint8) var46.x4[2]; + var54.x4[3] = (orc_uint8) var46.x4[3]; + /* 10: mullw */ + var55.x4[0] = (var54.x4[0] * var53.x4[0]) & 0xffff; + var55.x4[1] = (var54.x4[1] * var53.x4[1]) & 0xffff; + var55.x4[2] = (var54.x4[2] * var53.x4[2]) & 0xffff; + var55.x4[3] = (var54.x4[3] * var53.x4[3]) & 0xffff; + /* 12: convubw */ + var57.x4[0] = (orc_uint8) var56.x4[0]; + var57.x4[1] = (orc_uint8) var56.x4[1]; + var57.x4[2] = (orc_uint8) var56.x4[2]; + var57.x4[3] = (orc_uint8) var56.x4[3]; + /* 13: subw */ + var58.x4[0] = var57.x4[0] - var53.x4[0]; + var58.x4[1] = var57.x4[1] - var53.x4[1]; + var58.x4[2] = var57.x4[2] - var53.x4[2]; + var58.x4[3] = var57.x4[3] - var53.x4[3]; + /* 14: loadl */ + var59 = ptr0[i]; + /* 15: shrul */ + var60.i = ((orc_uint32) var59.i) >> 24; + /* 16: convlw */ + var61.i = var60.i; + /* 17: convwb */ + var62 = var61.i; + /* 18: splatbl */ + var63.i = + ((((orc_uint32) var62) & 0xff) << 24) | ((((orc_uint32) var62) & 0xff) + << 16) | ((((orc_uint32) var62) & 0xff) << 8) | (((orc_uint32) var62) + & 0xff); + /* 19: convubw */ + var64.x4[0] = (orc_uint8) var63.x4[0]; + var64.x4[1] = (orc_uint8) var63.x4[1]; + var64.x4[2] = (orc_uint8) var63.x4[2]; + var64.x4[3] = (orc_uint8) var63.x4[3]; + /* 20: mullw */ + var65.x4[0] = (var64.x4[0] * var58.x4[0]) & 0xffff; + var65.x4[1] = (var64.x4[1] * var58.x4[1]) & 0xffff; + var65.x4[2] = (var64.x4[2] * var58.x4[2]) & 0xffff; + var65.x4[3] = (var64.x4[3] * var58.x4[3]) & 0xffff; + /* 21: div255w */ + var66.x4[0] = + ((orc_uint16) (((orc_uint16) (var65.x4[0] + 128)) + + (((orc_uint16) (var65.x4[0] + 128)) >> 8))) >> 8; + var66.x4[1] = + ((orc_uint16) (((orc_uint16) (var65.x4[1] + 128)) + + (((orc_uint16) (var65.x4[1] + 128)) >> 8))) >> 8; + var66.x4[2] = + ((orc_uint16) (((orc_uint16) (var65.x4[2] + 128)) + + (((orc_uint16) (var65.x4[2] + 128)) >> 8))) >> 8; + var66.x4[3] = + ((orc_uint16) (((orc_uint16) (var65.x4[3] + 128)) + + (((orc_uint16) (var65.x4[3] + 128)) >> 8))) >> 8; + /* 22: convubw */ + var67.x4[0] = (orc_uint8) var59.x4[0]; + var67.x4[1] = (orc_uint8) var59.x4[1]; + var67.x4[2] = (orc_uint8) var59.x4[2]; + var67.x4[3] = (orc_uint8) var59.x4[3]; + /* 23: mullw */ + var68.x4[0] = (var67.x4[0] * var66.x4[0]) & 0xffff; + var68.x4[1] = (var67.x4[1] * var66.x4[1]) & 0xffff; + var68.x4[2] = (var67.x4[2] * var66.x4[2]) & 0xffff; + var68.x4[3] = (var67.x4[3] * var66.x4[3]) & 0xffff; + /* 24: addw */ + var69.x4[0] = var68.x4[0] + var55.x4[0]; + var69.x4[1] = var68.x4[1] + var55.x4[1]; + var69.x4[2] = var68.x4[2] + var55.x4[2]; + var69.x4[3] = var68.x4[3] + var55.x4[3]; + /* 25: addw */ + var70.x4[0] = var66.x4[0] + var53.x4[0]; + var70.x4[1] = var66.x4[1] + var53.x4[1]; + var70.x4[2] = var66.x4[2] + var53.x4[2]; + var70.x4[3] = var66.x4[3] + var53.x4[3]; + /* 26: divluw */ + var71.x4[0] = + ((var70.x4[0] & 0xff) == + 0) ? 255 : ORC_CLAMP_UB (((orc_uint16) var69.x4[0]) / + ((orc_uint16) var70.x4[0] & 0xff)); + var71.x4[1] = + ((var70.x4[1] & 0xff) == + 0) ? 255 : ORC_CLAMP_UB (((orc_uint16) var69.x4[1]) / + ((orc_uint16) var70.x4[1] & 0xff)); + var71.x4[2] = + ((var70.x4[2] & 0xff) == + 0) ? 255 : ORC_CLAMP_UB (((orc_uint16) var69.x4[2]) / + ((orc_uint16) var70.x4[2] & 0xff)); + var71.x4[3] = + ((var70.x4[3] & 0xff) == + 0) ? 255 : ORC_CLAMP_UB (((orc_uint16) var69.x4[3]) / + ((orc_uint16) var70.x4[3] & 0xff)); + /* 27: loadl */ + var72 = ptr0[i]; + /* 28: shrul */ + var73.i = ((orc_uint32) var72.i) >> 24; + /* 29: convlw */ + var74.i = var73.i; + /* 30: convwb */ + var75 = var74.i; + /* 31: splatbl */ + var76.i = + ((((orc_uint32) var75) & 0xff) << 24) | ((((orc_uint32) var75) & 0xff) + << 16) | ((((orc_uint32) var75) & 0xff) << 8) | (((orc_uint32) var75) + & 0xff); + /* 32: convubw */ + var77.x4[0] = (orc_uint8) var76.x4[0]; + var77.x4[1] = (orc_uint8) var76.x4[1]; + var77.x4[2] = (orc_uint8) var76.x4[2]; + var77.x4[3] = (orc_uint8) var76.x4[3]; + /* 33: addw */ + var78.x4[0] = var77.x4[0] + var53.x4[0]; + var78.x4[1] = var77.x4[1] + var53.x4[1]; + var78.x4[2] = var77.x4[2] + var53.x4[2]; + var78.x4[3] = var77.x4[3] + var53.x4[3]; + /* 34: convwb */ + var79.x4[0] = var71.x4[0]; + var79.x4[1] = var71.x4[1]; + var79.x4[2] = var71.x4[2]; + var79.x4[3] = var71.x4[3]; + /* 36: andl */ + var80.i = var79.i & var44.i; + /* 37: convwb */ + var81.x4[0] = var78.x4[0]; + var81.x4[1] = var78.x4[1]; + var81.x4[2] = var78.x4[2]; + var81.x4[3] = var78.x4[3]; + /* 39: andl */ + var82.i = var81.i & var45.i; + /* 40: orl */ + var83.i = var80.i | var82.i; + /* 41: storel */ + ptr0[i] = var83; + } + } + +} + +void +compositor_orc_overlay_bgra_addition (guint8 * ORC_RESTRICT d1, int d1_stride, + const guint8 * ORC_RESTRICT s1, int s1_stride, int p1, int n, int m) +{ + OrcExecutor _ex, *ex = &_ex; + static volatile int p_inited = 0; + static OrcCode *c = 0; + void (*func) (OrcExecutor *); + + if (!p_inited) { + orc_once_mutex_lock (); + if (!p_inited) { + OrcProgram *p; + +#if 1 + static const orc_uint8 bc[] = { + 1, 7, 9, 36, 99, 111, 109, 112, 111, 115, 105, 116, 111, 114, 95, 111, + 114, 99, 95, 111, 118, 101, 114, 108, 97, 121, 95, 98, 103, 114, 97, 95, + 97, 100, 100, 105, 116, 105, 111, 110, 11, 4, 4, 12, 4, 4, 14, 4, + 255, 255, 255, 255, 14, 4, 0, 0, 0, 255, 14, 4, 255, 255, 255, 0, + 14, 4, 24, 0, 0, 0, 16, 2, 20, 4, 20, 4, 20, 2, 20, 1, + 20, 8, 20, 8, 20, 8, 20, 8, 20, 4, 20, 8, 20, 8, 113, 32, + 4, 126, 33, 32, 19, 163, 34, 33, 157, 35, 34, 152, 40, 35, 21, 2, + 150, 36, 40, 21, 2, 89, 36, 36, 24, 21, 2, 80, 36, 36, 21, 2, + 150, 42, 32, 21, 2, 89, 42, 42, 36, 115, 40, 16, 21, 2, 150, 37, + 40, 21, 2, 98, 37, 37, 36, 113, 32, 0, 126, 33, 32, 19, 163, 34, + 33, 157, 35, 34, 152, 40, 35, 21, 2, 150, 38, 40, 21, 2, 89, 38, + 38, 37, 21, 2, 80, 38, 38, 21, 2, 150, 41, 32, 21, 2, 89, 41, + 41, 38, 21, 2, 70, 41, 41, 42, 21, 2, 70, 38, 38, 36, 21, 2, + 81, 41, 41, 38, 113, 32, 0, 126, 33, 32, 19, 163, 34, 33, 157, 35, + 34, 152, 40, 35, 21, 2, 150, 39, 40, 21, 2, 70, 39, 39, 36, 21, + 2, 157, 32, 41, 106, 32, 32, 18, 21, 2, 157, 40, 39, 106, 40, 40, + 17, 123, 32, 32, 40, 128, 0, 32, 2, 0, + }; + p = orc_program_new_from_static_bytecode (bc); + orc_program_set_backup_function (p, + _backup_compositor_orc_overlay_bgra_addition); +#else + p = orc_program_new (); + orc_program_set_2d (p); + orc_program_set_name (p, "compositor_orc_overlay_bgra_addition"); + orc_program_set_backup_function (p, + _backup_compositor_orc_overlay_bgra_addition); + orc_program_add_destination (p, 4, "d1"); + orc_program_add_source (p, 4, "s1"); + orc_program_add_constant (p, 4, 0xffffffff, "c1"); + orc_program_add_constant (p, 4, 0xff000000, "c2"); + orc_program_add_constant (p, 4, 0x00ffffff, "c3"); + orc_program_add_constant (p, 4, 0x00000018, "c4"); + orc_program_add_parameter (p, 2, "p1"); + orc_program_add_temporary (p, 4, "t1"); + orc_program_add_temporary (p, 4, "t2"); + orc_program_add_temporary (p, 2, "t3"); + orc_program_add_temporary (p, 1, "t4"); + orc_program_add_temporary (p, 8, "t5"); + orc_program_add_temporary (p, 8, "t6"); + orc_program_add_temporary (p, 8, "t7"); + orc_program_add_temporary (p, 8, "t8"); + orc_program_add_temporary (p, 4, "t9"); + orc_program_add_temporary (p, 8, "t10"); + orc_program_add_temporary (p, 8, "t11"); + + orc_program_append_2 (p, "loadl", 0, ORC_VAR_T1, ORC_VAR_S1, ORC_VAR_D1, + ORC_VAR_D1); + orc_program_append_2 (p, "shrul", 0, ORC_VAR_T2, ORC_VAR_T1, ORC_VAR_C4, + ORC_VAR_D1); + orc_program_append_2 (p, "convlw", 0, ORC_VAR_T3, ORC_VAR_T2, ORC_VAR_D1, + ORC_VAR_D1); + orc_program_append_2 (p, "convwb", 0, ORC_VAR_T4, ORC_VAR_T3, ORC_VAR_D1, + ORC_VAR_D1); + orc_program_append_2 (p, "splatbl", 0, ORC_VAR_T9, ORC_VAR_T4, ORC_VAR_D1, + ORC_VAR_D1); + orc_program_append_2 (p, "convubw", 2, ORC_VAR_T5, ORC_VAR_T9, ORC_VAR_D1, + ORC_VAR_D1); + orc_program_append_2 (p, "mullw", 2, ORC_VAR_T5, ORC_VAR_T5, ORC_VAR_P1, + ORC_VAR_D1); + orc_program_append_2 (p, "div255w", 2, ORC_VAR_T5, ORC_VAR_T5, ORC_VAR_D1, + ORC_VAR_D1); + orc_program_append_2 (p, "convubw", 2, ORC_VAR_T11, ORC_VAR_T1, + ORC_VAR_D1, ORC_VAR_D1); + orc_program_append_2 (p, "mullw", 2, ORC_VAR_T11, ORC_VAR_T11, ORC_VAR_T5, + ORC_VAR_D1); + orc_program_append_2 (p, "loadpl", 0, ORC_VAR_T9, ORC_VAR_C1, ORC_VAR_D1, + ORC_VAR_D1); + orc_program_append_2 (p, "convubw", 2, ORC_VAR_T6, ORC_VAR_T9, ORC_VAR_D1, + ORC_VAR_D1); + orc_program_append_2 (p, "subw", 2, ORC_VAR_T6, ORC_VAR_T6, ORC_VAR_T5, + ORC_VAR_D1); + orc_program_append_2 (p, "loadl", 0, ORC_VAR_T1, ORC_VAR_D1, ORC_VAR_D1, + ORC_VAR_D1); + orc_program_append_2 (p, "shrul", 0, ORC_VAR_T2, ORC_VAR_T1, ORC_VAR_C4, + ORC_VAR_D1); + orc_program_append_2 (p, "convlw", 0, ORC_VAR_T3, ORC_VAR_T2, ORC_VAR_D1, + ORC_VAR_D1); + orc_program_append_2 (p, "convwb", 0, ORC_VAR_T4, ORC_VAR_T3, ORC_VAR_D1, + ORC_VAR_D1); + orc_program_append_2 (p, "splatbl", 0, ORC_VAR_T9, ORC_VAR_T4, ORC_VAR_D1, + ORC_VAR_D1); + orc_program_append_2 (p, "convubw", 2, ORC_VAR_T7, ORC_VAR_T9, ORC_VAR_D1, + ORC_VAR_D1); + orc_program_append_2 (p, "mullw", 2, ORC_VAR_T7, ORC_VAR_T7, ORC_VAR_T6, + ORC_VAR_D1); + orc_program_append_2 (p, "div255w", 2, ORC_VAR_T7, ORC_VAR_T7, ORC_VAR_D1, + ORC_VAR_D1); + orc_program_append_2 (p, "convubw", 2, ORC_VAR_T10, ORC_VAR_T1, + ORC_VAR_D1, ORC_VAR_D1); + orc_program_append_2 (p, "mullw", 2, ORC_VAR_T10, ORC_VAR_T10, ORC_VAR_T7, + ORC_VAR_D1); + orc_program_append_2 (p, "addw", 2, ORC_VAR_T10, ORC_VAR_T10, ORC_VAR_T11, + ORC_VAR_D1); + orc_program_append_2 (p, "addw", 2, ORC_VAR_T7, ORC_VAR_T7, ORC_VAR_T5, + ORC_VAR_D1); + orc_program_append_2 (p, "divluw", 2, ORC_VAR_T10, ORC_VAR_T10, + ORC_VAR_T7, ORC_VAR_D1); + orc_program_append_2 (p, "loadl", 0, ORC_VAR_T1, ORC_VAR_D1, ORC_VAR_D1, + ORC_VAR_D1); + orc_program_append_2 (p, "shrul", 0, ORC_VAR_T2, ORC_VAR_T1, ORC_VAR_C4, + ORC_VAR_D1); + orc_program_append_2 (p, "convlw", 0, ORC_VAR_T3, ORC_VAR_T2, ORC_VAR_D1, + ORC_VAR_D1); + orc_program_append_2 (p, "convwb", 0, ORC_VAR_T4, ORC_VAR_T3, ORC_VAR_D1, + ORC_VAR_D1); + orc_program_append_2 (p, "splatbl", 0, ORC_VAR_T9, ORC_VAR_T4, ORC_VAR_D1, + ORC_VAR_D1); + orc_program_append_2 (p, "convubw", 2, ORC_VAR_T8, ORC_VAR_T9, ORC_VAR_D1, + ORC_VAR_D1); + orc_program_append_2 (p, "addw", 2, ORC_VAR_T8, ORC_VAR_T8, ORC_VAR_T5, + ORC_VAR_D1); + orc_program_append_2 (p, "convwb", 2, ORC_VAR_T1, ORC_VAR_T10, ORC_VAR_D1, + ORC_VAR_D1); + orc_program_append_2 (p, "andl", 0, ORC_VAR_T1, ORC_VAR_T1, ORC_VAR_C3, + ORC_VAR_D1); + orc_program_append_2 (p, "convwb", 2, ORC_VAR_T9, ORC_VAR_T8, ORC_VAR_D1, + ORC_VAR_D1); + orc_program_append_2 (p, "andl", 0, ORC_VAR_T9, ORC_VAR_T9, ORC_VAR_C2, + ORC_VAR_D1); + orc_program_append_2 (p, "orl", 0, ORC_VAR_T1, ORC_VAR_T1, ORC_VAR_T9, + ORC_VAR_D1); + orc_program_append_2 (p, "storel", 0, ORC_VAR_D1, ORC_VAR_T1, ORC_VAR_D1, + ORC_VAR_D1); +#endif + + orc_program_compile (p); + c = orc_program_take_code (p); + orc_program_free (p); + } + p_inited = TRUE; + orc_once_mutex_unlock (); + } + ex->arrays[ORC_VAR_A2] = c; + ex->program = 0; + + ex->n = n; + ORC_EXECUTOR_M (ex) = m; + ex->arrays[ORC_VAR_D1] = d1; + ex->params[ORC_VAR_D1] = d1_stride; + ex->arrays[ORC_VAR_S1] = (void *) s1; + ex->params[ORC_VAR_S1] = s1_stride; + ex->params[ORC_VAR_P1] = p1; + + func = c->exec; + func (ex); +} +#endif diff --git a/gst/compositor/compositororc-dist.h b/gst/compositor/compositororc-dist.h index 907b262b15..ae6f17ac8d 100644 --- a/gst/compositor/compositororc-dist.h +++ b/gst/compositor/compositororc-dist.h @@ -86,7 +86,9 @@ void compositor_orc_blend_u8 (guint8 * ORC_RESTRICT d1, int d1_stride, const gui void compositor_orc_blend_argb (guint8 * ORC_RESTRICT d1, int d1_stride, const guint8 * ORC_RESTRICT s1, int s1_stride, int p1, int n, int m); void compositor_orc_blend_bgra (guint8 * ORC_RESTRICT d1, int d1_stride, const guint8 * ORC_RESTRICT s1, int s1_stride, int p1, int n, int m); void compositor_orc_overlay_argb (guint8 * ORC_RESTRICT d1, int d1_stride, const guint8 * ORC_RESTRICT s1, int s1_stride, int p1, int n, int m); +void compositor_orc_overlay_argb_addition (guint8 * ORC_RESTRICT d1, int d1_stride, const guint8 * ORC_RESTRICT s1, int s1_stride, int p1, int n, int m); void compositor_orc_overlay_bgra (guint8 * ORC_RESTRICT d1, int d1_stride, const guint8 * ORC_RESTRICT s1, int s1_stride, int p1, int n, int m); +void compositor_orc_overlay_bgra_addition (guint8 * ORC_RESTRICT d1, int d1_stride, const guint8 * ORC_RESTRICT s1, int s1_stride, int p1, int n, int m); #ifdef __cplusplus } From afb652c169940ddda28e306f9792c4bad5685229 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim-Philipp=20M=C3=BCller?= Date: Thu, 3 Aug 2017 20:21:17 +0100 Subject: [PATCH 310/381] examples: fix compiler warning in compositor crossfade example warning: control reaches end of non-void function --- tests/examples/compositor/crossfade.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/examples/compositor/crossfade.c b/tests/examples/compositor/crossfade.c index de8dd45c79..ea2bb42ec2 100644 --- a/tests/examples/compositor/crossfade.c +++ b/tests/examples/compositor/crossfade.c @@ -132,4 +132,6 @@ main (int argc, char *argv[]) gst_print ("Timeout\n"); gst_element_set_state (pipeline, GST_STATE_NULL); gst_object_unref (pipeline); + + return 0; } From 088a5cec643bbd42d32aa7caa1dc8e8560e7c1c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Fri, 4 Aug 2017 11:08:18 +0300 Subject: [PATCH 311/381] videoaggregator: Don't mix up width and height CID 1416129 --- gst-libs/gst/video/gstvideoaggregator.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index 9864612acf..def0998d78 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -529,8 +529,8 @@ gst_video_aggregator_find_best_format (GstVideoAggregator * vagg, * Do not increment best_format_number in that case. */ gst_video_info_set_format (best_info, GST_VIDEO_FORMAT_ARGB, - GST_VIDEO_INFO_HEIGHT (&pad->info), - GST_VIDEO_INFO_WIDTH (&pad->info)); + GST_VIDEO_INFO_WIDTH (&pad->info), + GST_VIDEO_INFO_HEIGHT (&pad->info)); } else if (!need_alpha && (pad->info.finfo->flags & GST_VIDEO_FORMAT_FLAG_ALPHA)) { need_alpha = TRUE; From 663ef174d8190527bf94dab969198819b8bd80c7 Mon Sep 17 00:00:00 2001 From: Mathieu Duponchelle Date: Thu, 10 Aug 2017 01:43:15 +0200 Subject: [PATCH 312/381] compositor: improve conversion debugging https://bugzilla.gnome.org/show_bug.cgi?id=786078 --- gst/compositor/compositor.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/gst/compositor/compositor.c b/gst/compositor/compositor.c index dceca349c8..81744e34f0 100644 --- a/gst/compositor/compositor.c +++ b/gst/compositor/compositor.c @@ -314,9 +314,13 @@ gst_compositor_pad_set_info (GstVideoAggregatorPad * pad, tmp_info.flags = current_info->flags; tmp_info.interlace_mode = current_info->interlace_mode; - GST_DEBUG_OBJECT (pad, "This pad will be converted from %d to %d", - GST_VIDEO_INFO_FORMAT (current_info), - GST_VIDEO_INFO_FORMAT (&tmp_info)); + GST_DEBUG_OBJECT (pad, "This pad will be converted from format %s to %s, " + "colorimetry %s to %s, chroma-site %s to %s, " + "width/height %d/%d to %d/%d", + current_info->finfo->name, tmp_info.finfo->name, + colorimetry, best_colorimetry, + chroma, best_chroma, + current_info->width, current_info->height, width, height); cpad->convert = gst_video_converter_new (current_info, &tmp_info, NULL); cpad->conversion_info = tmp_info; From 7305f50512faa25fdb2ed198ff2249edb4d22505 Mon Sep 17 00:00:00 2001 From: Mathieu Duponchelle Date: Thu, 10 Aug 2017 01:45:53 +0200 Subject: [PATCH 313/381] videoaggregator: improve find_best_format heuristic. The goal here is to minimize the work needed to bring all images to a common format. A better criteria than the number of pads with a given format is the number of pixels with a given format. https://bugzilla.gnome.org/show_bug.cgi?id=786078 --- gst-libs/gst/video/gstvideoaggregator.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index def0998d78..a7b3bf8055 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -514,7 +514,7 @@ gst_video_aggregator_find_best_format (GstVideoAggregator * vagg, format_number = GPOINTER_TO_INT (g_hash_table_lookup (formats_table, GINT_TO_POINTER (GST_VIDEO_INFO_FORMAT (&pad->info)))); - format_number += 1; + format_number += pad->info.width * pad->info.height; g_hash_table_replace (formats_table, GINT_TO_POINTER (GST_VIDEO_INFO_FORMAT (&pad->info)), From 5322026885606ae1f7cee0fe99d1636c1e49500d Mon Sep 17 00:00:00 2001 From: Mathieu Duponchelle Date: Thu, 10 Aug 2017 01:48:18 +0200 Subject: [PATCH 314/381] videoaggregator: use colorimetry from find_best_format. This increases the chances that we won't need to do any conversion for a given pad. https://bugzilla.gnome.org/show_bug.cgi?id=786078 --- gst-libs/gst/video/gstvideoaggregator.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index a7b3bf8055..29ed69a431 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -636,13 +636,16 @@ gst_video_aggregator_default_update_caps (GstVideoAggregator * vagg, } GST_DEBUG_OBJECT (vagg, - "The output format will now be : %d with chroma : %s", - best_format, gst_video_chroma_to_string (best_info.chroma_site)); + "The output format will now be : %d with chroma : %s and colorimetry %s", + best_format, gst_video_chroma_to_string (best_info.chroma_site), + gst_video_colorimetry_to_string (&best_info.colorimetry)); best_format_caps = gst_caps_copy (caps); gst_caps_set_simple (best_format_caps, "format", G_TYPE_STRING, gst_video_format_to_string (best_format), "chroma-site", G_TYPE_STRING, - gst_video_chroma_to_string (best_info.chroma_site), NULL); + gst_video_chroma_to_string (best_info.chroma_site), "colorimetry", + G_TYPE_STRING, gst_video_colorimetry_to_string (&best_info.colorimetry), + NULL); ret = gst_caps_merge (best_format_caps, gst_caps_ref (caps)); return ret; From 3573238683a8ad1960a1e10c3400210a7c6cbc60 Mon Sep 17 00:00:00 2001 From: Julien Isorce Date: Fri, 7 Jul 2017 16:15:12 +0100 Subject: [PATCH 315/381] gl: do not include GL headers in public gstgl headers Except for gst/gl/gstglfuncs.h It is up to the client app to include these headers. It is coherent with the fact that gstreamer-gl.pc does not require any egl.pc/gles.pc. I.e. it is the responsability of the app to search these headers within its build setup. For example gstreamer-vaapi includes explicitly EGL/egl.h and search for it in its configure.ac. For example with this patch, if an app includes the headers gst/gl/egl/gstglcontext_egl.h gst/gl/egl/gstgldisplay_egl.h gst/gl/egl/gstglmemoryegl.h it will *no longer* automatically include EGL/egl.h and GLES2/gl2.h. Which is good because the app might want to use the gstgl api only without the need to bother about gl headers. Also added a test: cd tests/check && make libs/gstglheaders.check https://bugzilla.gnome.org/show_bug.cgi?id=784779 --- ext/gl/gstglmixer.h | 1 + ext/gl/gstglvideomixer.c | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/ext/gl/gstglmixer.h b/ext/gl/gstglmixer.h index 309f6bbf0c..886e00940a 100644 --- a/ext/gl/gstglmixer.h +++ b/ext/gl/gstglmixer.h @@ -24,6 +24,7 @@ #include #include #include +#include #include "gstglbasemixer.h" G_BEGIN_DECLS diff --git a/ext/gl/gstglvideomixer.c b/ext/gl/gstglvideomixer.c index a5a094b67e..7180220ded 100644 --- a/ext/gl/gstglvideomixer.c +++ b/ext/gl/gstglvideomixer.c @@ -43,10 +43,12 @@ #include "config.h" #endif -#include #include +#include +#include #include "gstglvideomixer.h" + #include "gstglmixerbin.h" #include "gstglutils.h" From c1f524f3a8ff9f3dc06dac257e8d521f8375ab64 Mon Sep 17 00:00:00 2001 From: Nicolas Dufresne Date: Tue, 5 Sep 2017 16:20:44 -0400 Subject: [PATCH 316/381] Request minimum buffer even if need_pool is FALSE When tee is used, it will not request a pool, but still it wants to know how many buffers are required. https://bugzilla.gnome.org/show_bug.cgi?id=730758 --- ext/gl/gstglmixer.c | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/ext/gl/gstglmixer.c b/ext/gl/gstglmixer.c index c0486616b2..af9de452b7 100644 --- a/ext/gl/gstglmixer.c +++ b/ext/gl/gstglmixer.c @@ -140,10 +140,10 @@ gst_gl_mixer_propose_allocation (GstAggregator * agg, GstBufferPool *pool = NULL; GstStructure *config; GstCaps *caps; + GstVideoInfo info; guint size = 0; gboolean need_pool; - if (!GST_AGGREGATOR_CLASS (gst_gl_mixer_parent_class)->propose_allocation (agg, agg_pad, decide_query, query)) return FALSE; @@ -155,18 +155,16 @@ gst_gl_mixer_propose_allocation (GstAggregator * agg, if (caps == NULL) goto no_caps; + if (!gst_video_info_from_caps (&info, caps)) + goto invalid_caps; + + /* the normal size of a frame */ + size = info.size; + if (need_pool) { - GstVideoInfo info; - - if (!gst_video_info_from_caps (&info, caps)) - goto invalid_caps; - GST_DEBUG_OBJECT (mix, "create new pool"); pool = gst_gl_buffer_pool_new (context); - /* the normal size of a frame */ - size = info.size; - config = gst_buffer_pool_get_config (pool); gst_buffer_pool_config_set_params (config, caps, size, 0, 0); @@ -174,11 +172,12 @@ gst_gl_mixer_propose_allocation (GstAggregator * agg, g_object_unref (pool); goto config_failed; } - - gst_query_add_allocation_pool (query, pool, size, 1, 0); - g_object_unref (pool); } + gst_query_add_allocation_pool (query, pool, size, 1, 0); + if (pool) + g_object_unref (pool); + /* we also support various metadata */ if (context->gl_vtable->FenceSync) gst_query_add_allocation_meta (query, GST_GL_SYNC_META_API_TYPE, 0); From 88fff68fc0e33bb061896253805440d27b8e6881 Mon Sep 17 00:00:00 2001 From: Haihua Hu Date: Mon, 18 Sep 2017 15:42:00 +0800 Subject: [PATCH 317/381] glvideomixer: need update output geometry after src caps reconfigure Need update output geometry when sink caps changed and use gst_structure_set to update caps if structure is fixed https://bugzilla.gnome.org/show_bug.cgi?id=787820 --- ext/gl/gstglvideomixer.c | 17 ++++++++++++----- ext/gl/gstglvideomixer.h | 2 ++ 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/ext/gl/gstglvideomixer.c b/ext/gl/gstglvideomixer.c index 7180220ded..45a85a734b 100644 --- a/ext/gl/gstglvideomixer.c +++ b/ext/gl/gstglvideomixer.c @@ -1076,6 +1076,7 @@ _fixate_caps (GstAggregator * agg, GstCaps * caps) best_fps_d = fps_d; } } + mix->output_geo_changed = TRUE; GST_OBJECT_UNLOCK (vagg); if (best_fps_n <= 0 || best_fps_d <= 0 || best_fps == 0.0) { @@ -1085,10 +1086,14 @@ _fixate_caps (GstAggregator * agg, GstCaps * caps) } s = gst_caps_get_structure (ret, 0); - gst_structure_fixate_field_nearest_int (s, "width", best_width); - gst_structure_fixate_field_nearest_int (s, "height", best_height); - gst_structure_fixate_field_nearest_fraction (s, "framerate", best_fps_n, - best_fps_d); + if (!gst_structure_fixate_field_nearest_int (s, "width", best_width)) + gst_structure_set (s, "width", G_TYPE_INT, best_width, NULL); + if (!gst_structure_fixate_field_nearest_int (s, "height", best_height)) + gst_structure_set (s, "height", G_TYPE_INT, best_height, NULL); + if (!gst_structure_fixate_field_nearest_fraction (s, "framerate", best_fps_n, + best_fps_d)) + gst_structure_set (s, "framerate", GST_TYPE_FRACTION, best_fps_n, + best_fps_d, NULL); ret = gst_caps_fixate (ret); return ret; @@ -1483,7 +1488,8 @@ gst_gl_video_mixer_callback (gpointer stuff) _init_vbo_indices (video_mixer); - if (pad->geometry_change || !pad->vertex_buffer) { + if (video_mixer->output_geo_changed || pad->geometry_change + || !pad->vertex_buffer) { gint pad_width, pad_height; gfloat w, h; @@ -1551,6 +1557,7 @@ gst_gl_video_mixer_callback (gpointer stuff) walk = g_list_next (walk); } + video_mixer->output_geo_changed = FALSE; GST_OBJECT_UNLOCK (video_mixer); gl->DisableVertexAttribArray (attr_position_loc); diff --git a/ext/gl/gstglvideomixer.h b/ext/gl/gstglvideomixer.h index f3526465eb..c734efbd94 100644 --- a/ext/gl/gstglvideomixer.h +++ b/ext/gl/gstglvideomixer.h @@ -126,6 +126,8 @@ struct _GstGLVideoMixer GLuint vbo_indices; GLuint checker_vbo; GstGLMemory *out_tex; + + gboolean output_geo_changed; }; struct _GstGLVideoMixerClass From 12c8a7c0afad03d4fa3af66af4ab030cfb3dbb16 Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Thu, 21 Sep 2017 11:59:22 +1000 Subject: [PATCH 318/381] Revert "glvideomixer: need update output geometry after src caps reconfigure" This reverts commit d6e538dc5651fb03c85d7c7614bcf6c689f2db2f. --- ext/gl/gstglvideomixer.c | 17 +++++------------ ext/gl/gstglvideomixer.h | 2 -- 2 files changed, 5 insertions(+), 14 deletions(-) diff --git a/ext/gl/gstglvideomixer.c b/ext/gl/gstglvideomixer.c index 45a85a734b..7180220ded 100644 --- a/ext/gl/gstglvideomixer.c +++ b/ext/gl/gstglvideomixer.c @@ -1076,7 +1076,6 @@ _fixate_caps (GstAggregator * agg, GstCaps * caps) best_fps_d = fps_d; } } - mix->output_geo_changed = TRUE; GST_OBJECT_UNLOCK (vagg); if (best_fps_n <= 0 || best_fps_d <= 0 || best_fps == 0.0) { @@ -1086,14 +1085,10 @@ _fixate_caps (GstAggregator * agg, GstCaps * caps) } s = gst_caps_get_structure (ret, 0); - if (!gst_structure_fixate_field_nearest_int (s, "width", best_width)) - gst_structure_set (s, "width", G_TYPE_INT, best_width, NULL); - if (!gst_structure_fixate_field_nearest_int (s, "height", best_height)) - gst_structure_set (s, "height", G_TYPE_INT, best_height, NULL); - if (!gst_structure_fixate_field_nearest_fraction (s, "framerate", best_fps_n, - best_fps_d)) - gst_structure_set (s, "framerate", GST_TYPE_FRACTION, best_fps_n, - best_fps_d, NULL); + gst_structure_fixate_field_nearest_int (s, "width", best_width); + gst_structure_fixate_field_nearest_int (s, "height", best_height); + gst_structure_fixate_field_nearest_fraction (s, "framerate", best_fps_n, + best_fps_d); ret = gst_caps_fixate (ret); return ret; @@ -1488,8 +1483,7 @@ gst_gl_video_mixer_callback (gpointer stuff) _init_vbo_indices (video_mixer); - if (video_mixer->output_geo_changed || pad->geometry_change - || !pad->vertex_buffer) { + if (pad->geometry_change || !pad->vertex_buffer) { gint pad_width, pad_height; gfloat w, h; @@ -1557,7 +1551,6 @@ gst_gl_video_mixer_callback (gpointer stuff) walk = g_list_next (walk); } - video_mixer->output_geo_changed = FALSE; GST_OBJECT_UNLOCK (video_mixer); gl->DisableVertexAttribArray (attr_position_loc); diff --git a/ext/gl/gstglvideomixer.h b/ext/gl/gstglvideomixer.h index c734efbd94..f3526465eb 100644 --- a/ext/gl/gstglvideomixer.h +++ b/ext/gl/gstglvideomixer.h @@ -126,8 +126,6 @@ struct _GstGLVideoMixer GLuint vbo_indices; GLuint checker_vbo; GstGLMemory *out_tex; - - gboolean output_geo_changed; }; struct _GstGLVideoMixerClass From 0fc911c7fe860548d1dbda002ccd07e718e42f46 Mon Sep 17 00:00:00 2001 From: Haihua Hu Date: Wed, 20 Sep 2017 20:00:03 +0800 Subject: [PATCH 319/381] glvidemixer: need reconfigure output gemotry after caps renegotiated --- ext/gl/gstglvideomixer.c | 8 +++++++- ext/gl/gstglvideomixer.h | 2 ++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/ext/gl/gstglvideomixer.c b/ext/gl/gstglvideomixer.c index 7180220ded..1bef5fd7d5 100644 --- a/ext/gl/gstglvideomixer.c +++ b/ext/gl/gstglvideomixer.c @@ -1161,6 +1161,9 @@ gst_gl_video_mixer_init_shader (GstGLMixer * mixer, GstCaps * outcaps) if (video_mixer->shader) gst_object_unref (video_mixer->shader); + /* need reconfigure output geometry */ + video_mixer->output_geo_change = TRUE; + return gst_gl_context_gen_shader (GST_GL_BASE_MIXER (mixer)->context, gst_gl_shader_string_vertex_mat4_vertex_transform, video_mixer_f_src, &video_mixer->shader); @@ -1483,7 +1486,8 @@ gst_gl_video_mixer_callback (gpointer stuff) _init_vbo_indices (video_mixer); - if (pad->geometry_change || !pad->vertex_buffer) { + if (video_mixer->output_geo_change + || pad->geometry_change || !pad->vertex_buffer) { gint pad_width, pad_height; gfloat w, h; @@ -1551,6 +1555,8 @@ gst_gl_video_mixer_callback (gpointer stuff) walk = g_list_next (walk); } + + video_mixer->output_geo_change = FALSE; GST_OBJECT_UNLOCK (video_mixer); gl->DisableVertexAttribArray (attr_position_loc); diff --git a/ext/gl/gstglvideomixer.h b/ext/gl/gstglvideomixer.h index f3526465eb..d07e1880a0 100644 --- a/ext/gl/gstglvideomixer.h +++ b/ext/gl/gstglvideomixer.h @@ -126,6 +126,8 @@ struct _GstGLVideoMixer GLuint vbo_indices; GLuint checker_vbo; GstGLMemory *out_tex; + + gboolean output_geo_change; }; struct _GstGLVideoMixerClass From 0ffa10652afc96cb267fa49a02039820f783eeea Mon Sep 17 00:00:00 2001 From: Ponnam Srinivas Date: Mon, 25 Sep 2017 17:20:58 +0530 Subject: [PATCH 320/381] glmixer: Unmap video frame in error case https://bugzilla.gnome.org/show_bug.cgi?id=788127 --- ext/gl/gstglmixer.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ext/gl/gstglmixer.c b/ext/gl/gstglmixer.c index af9de452b7..a4742a6d3d 100644 --- a/ext/gl/gstglmixer.c +++ b/ext/gl/gstglmixer.c @@ -616,8 +616,10 @@ gst_gl_mixer_process_textures (GstGLMixer * mix, GstBuffer * outbuf) out_tex = (GstGLMemory *) out_frame.map[0].memory; if (!gst_aggregator_iterate_sinkpads (GST_AGGREGATOR (mix), - (GstAggregatorPadForeachFunc) _upload_frames, NULL)) - return FALSE; + (GstAggregatorPadForeachFunc) _upload_frames, NULL)) { + res = FALSE; + goto out; + } g_mutex_lock (&priv->gl_resource_lock); if (!priv->gl_resource_ready) From 6c79262b6ee3797d6b69e438f5e30e8667c18f20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim-Philipp=20M=C3=BCller?= Date: Fri, 13 Oct 2017 11:46:09 +0100 Subject: [PATCH 321/381] videoaggregator: use GstFlowReturn for flow returns --- gst-libs/gst/video/gstvideoaggregator.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index 29ed69a431..dc86e84999 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -1046,7 +1046,7 @@ gst_video_aggregator_reset (GstVideoAggregator * vagg) GST_OBJECT_UNLOCK (vagg); } -static gint +static GstFlowReturn gst_video_aggregator_fill_queues (GstVideoAggregator * vagg, GstClockTime output_start_running_time, GstClockTime output_end_running_time) From eca9739bca20a310b0ac194eb7b0adaa42488043 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim-Philipp=20M=C3=BCller?= Date: Thu, 2 Nov 2017 11:57:24 +0000 Subject: [PATCH 322/381] videoaggregator: drop ABI compat in padding for new struct member Don't really have to do that while it's in -bad and most users are in-tree anyway. --- gst-libs/gst/video/gstvideoaggregator.c | 2 +- gst/compositor/compositor.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index dc86e84999..436c88a93d 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -521,7 +521,7 @@ gst_video_aggregator_find_best_format (GstVideoAggregator * vagg, GINT_TO_POINTER (format_number)); /* If that pad is the first with alpha, set it as the new best format */ - if (!need_alpha && (pad->ABI.needs_alpha + if (!need_alpha && (pad->needs_alpha && (!GST_VIDEO_FORMAT_INFO_HAS_ALPHA (pad->info.finfo)))) { need_alpha = TRUE; /* Just fallback to ARGB in case we require alpha but the input pad diff --git a/gst/compositor/compositor.c b/gst/compositor/compositor.c index 81744e34f0..c1954e2a69 100644 --- a/gst/compositor/compositor.c +++ b/gst/compositor/compositor.c @@ -195,7 +195,7 @@ gst_compositor_pad_set_property (GObject * object, guint prop_id, break; case PROP_PAD_CROSSFADE_RATIO: pad->crossfade = g_value_get_double (value); - GST_VIDEO_AGGREGATOR_PAD (pad)->ABI.needs_alpha = pad->crossfade >= 0.0f; + GST_VIDEO_AGGREGATOR_PAD (pad)->needs_alpha = pad->crossfade >= 0.0f; break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); From b798cfbece9d33601b315fa16993c5c18318ea05 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim-Philipp=20M=C3=BCller?= Date: Thu, 2 Nov 2017 12:17:38 +0000 Subject: [PATCH 323/381] gl: use new gst_element_foreach_sink_pad() Instead of gst_aggregator_iterate_sinkpads() which will soon be removed. https://bugzilla.gnome.org/show_bug.cgi?id=785679 --- ext/gl/gstglmixer.c | 14 +++++++------- ext/gl/gstglvideomixer.c | 5 ++--- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/ext/gl/gstglmixer.c b/ext/gl/gstglmixer.c index a4742a6d3d..06b50134c5 100644 --- a/ext/gl/gstglmixer.c +++ b/ext/gl/gstglmixer.c @@ -561,12 +561,12 @@ context_error: } static gboolean -_upload_frames (GstAggregator * agg, GstAggregatorPad * agg_pad, +gst_gl_mixer_upload_frames (GstElement * element, GstPad * sink_pad, gpointer user_data) { - GstVideoAggregatorPad *vaggpad = GST_VIDEO_AGGREGATOR_PAD (agg_pad); - GstGLMixerPad *pad = GST_GL_MIXER_PAD (agg_pad); - GstGLMixer *mix = GST_GL_MIXER (agg); + GstVideoAggregatorPad *vaggpad = GST_VIDEO_AGGREGATOR_PAD (sink_pad); + GstGLMixerPad *pad = GST_GL_MIXER_PAD (sink_pad); + GstGLMixer *mix = GST_GL_MIXER (element); pad->current_texture = 0; if (vaggpad->buffer != NULL) { @@ -585,7 +585,7 @@ _upload_frames (GstAggregator * agg, GstAggregatorPad * agg_pad, if (!gst_video_frame_map (&gl_frame, &gl_info, vaggpad->buffer, GST_MAP_READ | GST_MAP_GL)) { - GST_ERROR_OBJECT (agg_pad, "Failed to map input frame"); + GST_ERROR_OBJECT (pad, "Failed to map input frame"); return FALSE; } @@ -615,8 +615,8 @@ gst_gl_mixer_process_textures (GstGLMixer * mix, GstBuffer * outbuf) out_tex = (GstGLMemory *) out_frame.map[0].memory; - if (!gst_aggregator_iterate_sinkpads (GST_AGGREGATOR (mix), - (GstAggregatorPadForeachFunc) _upload_frames, NULL)) { + if (!gst_element_foreach_sink_pad (GST_ELEMENT_CAST (mix), + gst_gl_mixer_upload_frames, NULL)) { res = FALSE; goto out; } diff --git a/ext/gl/gstglvideomixer.c b/ext/gl/gstglvideomixer.c index 1bef5fd7d5..91e5ad2632 100644 --- a/ext/gl/gstglvideomixer.c +++ b/ext/gl/gstglvideomixer.c @@ -1095,7 +1095,7 @@ _fixate_caps (GstAggregator * agg, GstCaps * caps) } static gboolean -_reset_pad_gl (GstAggregator * agg, GstAggregatorPad * aggpad, gpointer udata) +_reset_pad_gl (GstElement * agg, GstPad * aggpad, gpointer udata) { const GstGLFuncs *gl = GST_GL_BASE_MIXER (agg)->context->gl_vtable; GstGLVideoMixerPad *pad = GST_GL_VIDEO_MIXER_PAD (aggpad); @@ -1128,8 +1128,7 @@ _reset_gl (GstGLContext * context, GstGLVideoMixer * video_mixer) video_mixer->checker_vbo = 0; } - gst_aggregator_iterate_sinkpads (GST_AGGREGATOR (video_mixer), _reset_pad_gl, - NULL); + gst_element_foreach_sink_pad (GST_ELEMENT (video_mixer), _reset_pad_gl, NULL); } static void From e42344a25c0aa225fc5694e7ccc57b4038be081d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim-Philipp=20M=C3=BCller?= Date: Thu, 2 Nov 2017 12:56:57 +0000 Subject: [PATCH 324/381] videoaggregator: add CAST macros --- gst-libs/gst/video/gstvideoaggregator.h | 1 + 1 file changed, 1 insertion(+) diff --git a/gst-libs/gst/video/gstvideoaggregator.h b/gst-libs/gst/video/gstvideoaggregator.h index 3c74d1aa84..9882a75dfe 100644 --- a/gst-libs/gst/video/gstvideoaggregator.h +++ b/gst-libs/gst/video/gstvideoaggregator.h @@ -35,6 +35,7 @@ G_BEGIN_DECLS #define GST_TYPE_VIDEO_AGGREGATOR (gst_video_aggregator_get_type()) #define GST_VIDEO_AGGREGATOR(obj) \ (G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_VIDEO_AGGREGATOR, GstVideoAggregator)) +#define GST_VIDEO_AGGREGATOR_CAST(obj) ((GstVideoAggregator *)(obj)) #define GST_VIDEO_AGGREGATOR_CLASS(klass) \ (G_TYPE_CHECK_CLASS_CAST((klass),GST_TYPE_VIDEO_AGGREGATOR, GstVideoAggregatorClass)) #define GST_IS_VIDEO_AGGREGATOR(obj) \ From 96e9eb1fd3e6bc587361da18c765fdaf8c8616fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim-Philipp=20M=C3=BCller?= Date: Thu, 2 Nov 2017 12:46:26 +0000 Subject: [PATCH 325/381] videoaggregator: use new gst_element_foreach_sink_pad() Instead of gst_aggregator_iterate_sinkpads() which will soon be removed. https://bugzilla.gnome.org/show_bug.cgi?id=785679 --- gst-libs/gst/video/gstvideoaggregator.c | 33 +++++++++++++------------ 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index 436c88a93d..e9b31579db 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -1261,16 +1261,17 @@ gst_video_aggregator_fill_queues (GstVideoAggregator * vagg, } static gboolean -sync_pad_values (GstVideoAggregator * vagg, GstVideoAggregatorPad * pad) +sync_pad_values (GstElement * vagg, GstPad * pad, gpointer user_data) { - GstAggregatorPad *bpad = GST_AGGREGATOR_PAD (pad); + GstVideoAggregatorPad *vpad = GST_VIDEO_AGGREGATOR_PAD (pad); + GstAggregatorPad *bpad = GST_AGGREGATOR_PAD_CAST (pad); GstClockTime timestamp; gint64 stream_time; - if (pad->buffer == NULL) + if (vpad->buffer == NULL) return TRUE; - timestamp = GST_BUFFER_TIMESTAMP (pad->buffer); + timestamp = GST_BUFFER_TIMESTAMP (vpad->buffer); GST_OBJECT_LOCK (bpad); stream_time = gst_segment_to_stream_time (&bpad->segment, GST_FORMAT_TIME, timestamp); @@ -1278,30 +1279,33 @@ sync_pad_values (GstVideoAggregator * vagg, GstVideoAggregatorPad * pad) /* sync object properties on stream time */ if (GST_CLOCK_TIME_IS_VALID (stream_time)) - gst_object_sync_values (GST_OBJECT (pad), stream_time); + gst_object_sync_values (GST_OBJECT_CAST (pad), stream_time); return TRUE; } static gboolean -prepare_frames (GstVideoAggregator * vagg, GstVideoAggregatorPad * pad) +prepare_frames (GstElement * agg, GstPad * pad, gpointer user_data) { + GstVideoAggregatorPad *vpad = GST_VIDEO_AGGREGATOR_PAD_CAST (pad); GstVideoAggregatorPadClass *vaggpad_class = GST_VIDEO_AGGREGATOR_PAD_GET_CLASS (pad); - if (pad->buffer == NULL || !vaggpad_class->prepare_frame) + if (vpad->buffer == NULL || !vaggpad_class->prepare_frame) return TRUE; - return vaggpad_class->prepare_frame (pad, vagg); + return vaggpad_class->prepare_frame (vpad, GST_VIDEO_AGGREGATOR_CAST (agg)); } static gboolean -clean_pad (GstVideoAggregator * vagg, GstVideoAggregatorPad * pad) +clean_pad (GstElement * agg, GstPad * pad, gpointer user_data) { + GstVideoAggregator *vagg = GST_VIDEO_AGGREGATOR_CAST (agg); + GstVideoAggregatorPad *vpad = GST_VIDEO_AGGREGATOR_PAD_CAST (pad); GstVideoAggregatorPadClass *vaggpad_class = GST_VIDEO_AGGREGATOR_PAD_GET_CLASS (pad); - vaggpad_class->clean_frame (pad, vagg); + vaggpad_class->clean_frame (vpad, vagg); return TRUE; } @@ -1334,18 +1338,15 @@ gst_video_aggregator_do_aggregate (GstVideoAggregator * vagg, GST_BUFFER_DURATION (*outbuf) = output_end_time - output_start_time; /* Sync pad properties to the stream time */ - gst_aggregator_iterate_sinkpads (GST_AGGREGATOR (vagg), - (GstAggregatorPadForeachFunc) sync_pad_values, NULL); + gst_element_foreach_sink_pad (GST_ELEMENT_CAST (vagg), sync_pad_values, NULL); /* Convert all the frames the subclass has before aggregating */ - gst_aggregator_iterate_sinkpads (GST_AGGREGATOR (vagg), - (GstAggregatorPadForeachFunc) prepare_frames, NULL); + gst_element_foreach_sink_pad (GST_ELEMENT_CAST (vagg), prepare_frames, NULL); ret = vagg_klass->aggregate_frames (vagg, *outbuf); if (vaggpad_class->clean_frame) { - gst_aggregator_iterate_sinkpads (GST_AGGREGATOR (vagg), - (GstAggregatorPadForeachFunc) clean_pad, NULL); + gst_element_foreach_sink_pad (GST_ELEMENT_CAST (vagg), clean_pad, NULL); } return ret; From 14f886b7b0d2baa6b70aafa0f606e5d46ce0123d Mon Sep 17 00:00:00 2001 From: Mathieu Duponchelle Date: Mon, 6 Nov 2017 21:07:51 +0100 Subject: [PATCH 326/381] aggregator: Remove klass->sinkpads_type This posed problems for the python bindings (and possibly others). Instead, subclasses now use add_pad_template_with_gtype. https://bugzilla.gnome.org/show_bug.cgi?id=789986 --- ext/gl/gstglbasemixer.c | 1 - ext/gl/gstglmixer.c | 4 ++-- ext/gl/gstglstereomix.c | 4 ++-- ext/gl/gstglvideomixer.c | 12 ++++++++++- gst-libs/gst/video/gstvideoaggregator.c | 27 ++++++++++--------------- gst/compositor/compositor.c | 4 ++-- 6 files changed, 28 insertions(+), 24 deletions(-) diff --git a/ext/gl/gstglbasemixer.c b/ext/gl/gstglbasemixer.c index 3e77e3394d..87cea2c4b2 100644 --- a/ext/gl/gstglbasemixer.c +++ b/ext/gl/gstglbasemixer.c @@ -270,7 +270,6 @@ gst_gl_base_mixer_class_init (GstGLBaseMixerClass * klass) GST_DEBUG_FUNCPTR (gst_gl_base_mixer_set_context); element_class->change_state = gst_gl_base_mixer_change_state; - agg_class->sinkpads_type = GST_TYPE_GL_BASE_MIXER_PAD; agg_class->sink_query = gst_gl_base_mixer_sink_query; agg_class->src_query = gst_gl_base_mixer_src_query; agg_class->src_activate = gst_gl_base_mixer_src_activate_mode; diff --git a/ext/gl/gstglmixer.c b/ext/gl/gstglmixer.c index 06b50134c5..856a433f9d 100644 --- a/ext/gl/gstglmixer.c +++ b/ext/gl/gstglmixer.c @@ -365,9 +365,9 @@ gst_gl_mixer_class_init (GstGLMixerClass * klass) gobject_class->set_property = gst_gl_mixer_set_property; gst_element_class_add_static_pad_template (element_class, &src_factory); - gst_element_class_add_static_pad_template (element_class, &sink_factory); + gst_element_class_add_static_pad_template_with_gtype (element_class, + &sink_factory, GST_TYPE_GL_MIXER_PAD); - agg_class->sinkpads_type = GST_TYPE_GL_MIXER_PAD; agg_class->sink_query = gst_gl_mixer_sink_query; agg_class->src_query = gst_gl_mixer_src_query; agg_class->stop = gst_gl_mixer_stop; diff --git a/ext/gl/gstglstereomix.c b/ext/gl/gstglstereomix.c index 9e42b23f67..d3fe0302f2 100644 --- a/ext/gl/gstglstereomix.c +++ b/ext/gl/gstglstereomix.c @@ -180,9 +180,9 @@ gst_gl_stereo_mix_class_init (GstGLStereoMixClass * klass) G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); gst_element_class_add_static_pad_template (element_class, &src_factory); - gst_element_class_add_static_pad_template (element_class, &sink_factory); + gst_element_class_add_static_pad_template_with_gtype (element_class, + &sink_factory, GST_TYPE_GL_STEREO_MIX_PAD); - agg_class->sinkpads_type = GST_TYPE_GL_STEREO_MIX_PAD; agg_class->stop = gst_gl_stereo_mix_stop; agg_class->start = gst_gl_stereo_mix_start; agg_class->src_query = gst_gl_stereo_mix_src_query; diff --git a/ext/gl/gstglvideomixer.c b/ext/gl/gstglvideomixer.c index 91e5ad2632..487bf2fe16 100644 --- a/ext/gl/gstglvideomixer.c +++ b/ext/gl/gstglvideomixer.c @@ -55,6 +55,14 @@ #define GST_CAT_DEFAULT gst_gl_video_mixer_debug GST_DEBUG_CATEGORY_STATIC (GST_CAT_DEFAULT); +static GstStaticPadTemplate sink_factory = GST_STATIC_PAD_TEMPLATE ("sink_%u", + GST_PAD_SINK, + GST_PAD_REQUEST, + GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES + (GST_CAPS_FEATURE_MEMORY_GL_MEMORY, + "RGBA")) + ); + #define GST_TYPE_GL_VIDEO_MIXER_BACKGROUND (gst_gl_video_mixer_background_get_type()) static GType gst_gl_video_mixer_background_get_type (void) @@ -862,6 +870,9 @@ gst_gl_video_mixer_class_init (GstGLVideoMixerClass * klass) "Filter/Effect/Video/Compositor", "OpenGL video_mixer", "Matthew Waters "); + gst_element_class_add_static_pad_template_with_gtype (element_class, + &sink_factory, GST_TYPE_GL_VIDEO_MIXER_PAD); + g_object_class_install_property (gobject_class, PROP_BACKGROUND, g_param_spec_enum ("background", "Background", "Background type", GST_TYPE_GL_VIDEO_MIXER_BACKGROUND, @@ -875,7 +886,6 @@ gst_gl_video_mixer_class_init (GstGLVideoMixerClass * klass) vagg_class->update_caps = _update_caps; - agg_class->sinkpads_type = GST_TYPE_GL_VIDEO_MIXER_PAD; agg_class->fixate_src_caps = _fixate_caps; agg_class->propose_allocation = gst_gl_video_mixer_propose_allocation; diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index e9b31579db..3cb812a4b0 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -692,8 +692,6 @@ static gboolean gst_video_aggregator_default_negotiated_src_caps (GstAggregator * agg, GstCaps * caps) { - GstVideoAggregatorPadClass *vaggpad_klass = g_type_class_peek - (GST_AGGREGATOR_GET_CLASS (agg)->sinkpads_type); GstVideoAggregator *vagg = GST_VIDEO_AGGREGATOR (agg); gboolean at_least_one_alpha = FALSE; const GstVideoFormatInfo *finfo; @@ -741,14 +739,15 @@ gst_video_aggregator_default_negotiated_src_caps (GstAggregator * agg, return FALSE; } - if (vaggpad_klass->set_info) { - /* Then browse the sinks once more, setting or unsetting conversion if needed */ - for (l = GST_ELEMENT (vagg)->sinkpads; l; l = l->next) { - GstVideoAggregatorPad *pad = GST_VIDEO_AGGREGATOR_PAD (l->data); + /* Then browse the sinks once more, setting or unsetting conversion if needed */ + for (l = GST_ELEMENT (vagg)->sinkpads; l; l = l->next) { + GstVideoAggregatorPad *pad = GST_VIDEO_AGGREGATOR_PAD (l->data); + GstVideoAggregatorPadClass *vaggpad_klass = + GST_VIDEO_AGGREGATOR_PAD_GET_CLASS (pad); - if (!vaggpad_klass->set_info (pad, vagg, &pad->info, &vagg->info)) { - return FALSE; - } + if (vaggpad_klass->set_info + && !vaggpad_klass->set_info (pad, vagg, &pad->info, &vagg->info)) { + return FALSE; } } @@ -1305,7 +1304,8 @@ clean_pad (GstElement * agg, GstPad * pad, gpointer user_data) GstVideoAggregatorPadClass *vaggpad_class = GST_VIDEO_AGGREGATOR_PAD_GET_CLASS (pad); - vaggpad_class->clean_frame (vpad, vagg); + if (vaggpad_class->clean_frame) + vaggpad_class->clean_frame (vpad, vagg); return TRUE; } @@ -1318,8 +1318,6 @@ gst_video_aggregator_do_aggregate (GstVideoAggregator * vagg, GstFlowReturn ret = GST_FLOW_OK; GstElementClass *klass = GST_ELEMENT_GET_CLASS (vagg); GstVideoAggregatorClass *vagg_klass = (GstVideoAggregatorClass *) klass; - GstVideoAggregatorPadClass *vaggpad_class = g_type_class_peek - (GST_AGGREGATOR_CLASS (klass)->sinkpads_type); g_assert (vagg_klass->aggregate_frames != NULL); g_assert (vagg_klass->get_output_buffer != NULL); @@ -1345,9 +1343,7 @@ gst_video_aggregator_do_aggregate (GstVideoAggregator * vagg, ret = vagg_klass->aggregate_frames (vagg, *outbuf); - if (vaggpad_class->clean_frame) { - gst_element_foreach_sink_pad (GST_ELEMENT_CAST (vagg), clean_pad, NULL); - } + gst_element_foreach_sink_pad (GST_ELEMENT_CAST (vagg), clean_pad, NULL); return ret; } @@ -2189,7 +2185,6 @@ gst_video_aggregator_class_init (GstVideoAggregatorClass * klass) gstelement_class->release_pad = GST_DEBUG_FUNCPTR (gst_video_aggregator_release_pad); - agg_class->sinkpads_type = GST_TYPE_VIDEO_AGGREGATOR_PAD; agg_class->start = gst_video_aggregator_start; agg_class->stop = gst_video_aggregator_stop; agg_class->sink_query = gst_video_aggregator_sink_query; diff --git a/gst/compositor/compositor.c b/gst/compositor/compositor.c index c1954e2a69..2980e5c407 100644 --- a/gst/compositor/compositor.c +++ b/gst/compositor/compositor.c @@ -1223,7 +1223,6 @@ gst_compositor_class_init (GstCompositorClass * klass) gobject_class->get_property = gst_compositor_get_property; gobject_class->set_property = gst_compositor_set_property; - agg_class->sinkpads_type = GST_TYPE_COMPOSITOR_PAD; agg_class->sink_query = _sink_query; agg_class->fixate_src_caps = _fixate_caps; agg_class->negotiated_src_caps = _negotiated_caps; @@ -1235,7 +1234,8 @@ gst_compositor_class_init (GstCompositorClass * klass) DEFAULT_BACKGROUND, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); gst_element_class_add_static_pad_template (gstelement_class, &src_factory); - gst_element_class_add_static_pad_template (gstelement_class, &sink_factory); + gst_element_class_add_static_pad_template_with_gtype (gstelement_class, + &sink_factory, GST_TYPE_COMPOSITOR_PAD); gst_element_class_set_static_metadata (gstelement_class, "Compositor", "Filter/Editor/Video/Compositor", From eb491b9a62b82c8b77f8b5682dbd701f7c409376 Mon Sep 17 00:00:00 2001 From: Edward Hervey Date: Sat, 25 Nov 2017 12:48:40 +0100 Subject: [PATCH 327/381] videoaggregator: Don't leak string The result of gst_video_colorimetry_to_string () needs to be free'd --- gst-libs/gst/video/gstvideoaggregator.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index 3cb812a4b0..00e02768f2 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -618,6 +618,7 @@ gst_video_aggregator_default_update_caps (GstVideoAggregator * vagg, gboolean at_least_one_alpha = FALSE; GstVideoFormat best_format; GstVideoInfo best_info; + gchar *color_name; best_format = GST_VIDEO_FORMAT_UNKNOWN; gst_video_info_init (&best_info); @@ -641,11 +642,12 @@ gst_video_aggregator_default_update_caps (GstVideoAggregator * vagg, gst_video_colorimetry_to_string (&best_info.colorimetry)); best_format_caps = gst_caps_copy (caps); + color_name = gst_video_colorimetry_to_string (&best_info.colorimetry); gst_caps_set_simple (best_format_caps, "format", G_TYPE_STRING, gst_video_format_to_string (best_format), "chroma-site", G_TYPE_STRING, gst_video_chroma_to_string (best_info.chroma_site), "colorimetry", - G_TYPE_STRING, gst_video_colorimetry_to_string (&best_info.colorimetry), - NULL); + G_TYPE_STRING, color_name, NULL); + g_free (color_name); ret = gst_caps_merge (best_format_caps, gst_caps_ref (caps)); return ret; From 2b933a28272848e6945ea297e9bdc74d2cde0f39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim-Philipp=20M=C3=BCller?= Date: Sat, 2 Dec 2017 16:01:25 +0000 Subject: [PATCH 328/381] Remove GstAggregator from -bad, moved to core https://bugzilla.gnome.org/show_bug.cgi?id=739010 --- gst/compositor/Makefile.am | 1 - gst/compositor/meson.build | 3 +-- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/gst/compositor/Makefile.am b/gst/compositor/Makefile.am index 5d8e184d47..6b98b78bbc 100644 --- a/gst/compositor/Makefile.am +++ b/gst/compositor/Makefile.am @@ -16,7 +16,6 @@ libgstcompositor_la_CFLAGS = \ $(GST_PLUGINS_BASE_CFLAGS) \ $(GST_BASE_CFLAGS) $(GST_CFLAGS) $(ORC_CFLAGS) libgstcompositor_la_LIBADD = \ - $(top_builddir)/gst-libs/gst/base/libgstbadbase-$(GST_API_VERSION).la \ $(top_builddir)/gst-libs/gst/video/libgstbadvideo-$(GST_API_VERSION).la \ $(GST_PLUGINS_BASE_LIBS) \ -lgstvideo-@GST_API_VERSION@ \ diff --git a/gst/compositor/meson.build b/gst/compositor/meson.build index 2e948fcb0c..52ca87385e 100644 --- a/gst/compositor/meson.build +++ b/gst/compositor/meson.build @@ -26,8 +26,7 @@ gstcompositor = library('gstcompositor', compositor_sources, orc_c, orc_h, c_args : gst_plugins_bad_args + ['-DGST_USE_UNSTABLE_API'], include_directories : [configinc], - dependencies : [gstbadvideo_dep, gstbadbase_dep, gstvideo_dep, gstbase_dep, - orc_dep, libm], + dependencies : [gstbadvideo_dep, gstvideo_dep, gstbase_dep, orc_dep, libm], install : true, install_dir : plugins_install_dir, ) From 944f74b834151b4dc23b36c11ef318946368ee7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim-Philipp=20M=C3=BCller?= Date: Tue, 23 Jan 2018 09:01:00 +0000 Subject: [PATCH 329/381] Update for renamed aggregator pad API https://bugzilla.gnome.org/show_bug.cgi?id=791204 --- gst-libs/gst/video/gstvideoaggregator.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index 00e02768f2..a0c23b7466 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -1077,7 +1077,7 @@ gst_video_aggregator_fill_queues (GstVideoAggregator * vagg, if (!is_eos) eos = FALSE; - buf = gst_aggregator_pad_get_buffer (bpad); + buf = gst_aggregator_pad_peek_buffer (bpad); if (buf) { GstClockTime start_time, end_time; @@ -1122,7 +1122,7 @@ gst_video_aggregator_fill_queues (GstVideoAggregator * vagg, continue; } gst_buffer_unref (buf); - buf = gst_aggregator_pad_steal_buffer (bpad); + buf = gst_aggregator_pad_pop_buffer (bpad); gst_buffer_replace (&pad->buffer, buf); if (pad->priv->pending_vinfo.finfo) { pad->info = pad->priv->pending_vinfo; From a4f609f8473e39f2f698c3092491f64573ad7d99 Mon Sep 17 00:00:00 2001 From: Mathieu Duponchelle Date: Thu, 28 Dec 2017 12:15:21 +0100 Subject: [PATCH 330/381] videoaggregatorpad: implement skip_buffer Skip buffers from sources with a framerate higher than the output framerate. https://bugzilla.gnome.org/show_bug.cgi?id=781928 --- gst-libs/gst/video/gstvideoaggregator.c | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index a0c23b7466..6c043782ba 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -219,6 +219,28 @@ gst_video_aggregator_pad_set_info (GstVideoAggregatorPad * pad, return TRUE; } +static gboolean +gst_video_aggregator_pad_skip_buffer (GstAggregatorPad * aggpad, + GstAggregator * agg, GstBuffer * buffer) +{ + gboolean ret = FALSE; + + if (agg->segment.position != GST_CLOCK_TIME_NONE + && GST_BUFFER_DURATION (buffer) != GST_CLOCK_TIME_NONE) { + GstClockTime start_time = + gst_segment_to_running_time (&aggpad->segment, GST_FORMAT_TIME, + GST_BUFFER_PTS (buffer)); + GstClockTime end_time = start_time + GST_BUFFER_DURATION (buffer); + GstClockTime output_start_running_time = + gst_segment_to_running_time (&agg->segment, GST_FORMAT_TIME, + agg->segment.position); + + ret = end_time < output_start_running_time; + } + + return ret; +} + static void gst_video_aggregator_pad_finalize (GObject * o) { @@ -324,6 +346,8 @@ gst_video_aggregator_pad_class_init (GstVideoAggregatorPadClass * klass) g_type_class_add_private (klass, sizeof (GstVideoAggregatorPadPrivate)); aggpadclass->flush = GST_DEBUG_FUNCPTR (_flush_pad); + aggpadclass->skip_buffer = + GST_DEBUG_FUNCPTR (gst_video_aggregator_pad_skip_buffer); klass->set_info = GST_DEBUG_FUNCPTR (gst_video_aggregator_pad_set_info); klass->prepare_frame = GST_DEBUG_FUNCPTR (gst_video_aggregator_pad_prepare_frame); From 6df4606044073affa9cd7ff191067044de2b4b67 Mon Sep 17 00:00:00 2001 From: Vincent Penquerc'h Date: Mon, 3 Oct 2016 13:11:07 +0100 Subject: [PATCH 331/381] glvideomixer: fix vertex_buffer leak We call the base class first as this will remove the pad from the aggregator, thus stopping misc callbacks from being called, one of which (process_textures) will recreate the vertex_buffer if it is destroyed https://bugzilla.gnome.org/show_bug.cgi?id=760873 --- ext/gl/gstglvideomixer.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/ext/gl/gstglvideomixer.c b/ext/gl/gstglvideomixer.c index 487bf2fe16..78b5d06c51 100644 --- a/ext/gl/gstglvideomixer.c +++ b/ext/gl/gstglvideomixer.c @@ -841,14 +841,20 @@ static void gst_gl_video_mixer_release_pad (GstElement * element, GstPad * p) { GstGLVideoMixerPad *pad = GST_GL_VIDEO_MIXER_PAD (p); + + /* we call the base class first as this will remove the pad from + * the aggregator, thus stopping misc callbacks from being called, + * one of which (process_textures) will recreate the vertex_buffer + * if it is destroyed */ + GST_ELEMENT_CLASS (g_type_class_peek_parent (G_OBJECT_GET_CLASS (element))) + ->release_pad (element, p); + if (pad->vertex_buffer) { GstGLBaseMixer *mix = GST_GL_BASE_MIXER (element); gst_gl_context_thread_add (mix->context, (GstGLContextThreadFunc) _del_buffer, &pad->vertex_buffer); pad->vertex_buffer = 0; } - GST_ELEMENT_CLASS (g_type_class_peek_parent (G_OBJECT_GET_CLASS (element))) - ->release_pad (element, p); } static void From 3b9843d13b9340850830de345cc47cb8492c20ec Mon Sep 17 00:00:00 2001 From: Edward Hervey Date: Wed, 14 Feb 2018 14:36:00 +0100 Subject: [PATCH 332/381] Update ORC fallback disted code --- gst/compositor/compositororc-dist.c | 60 ++++++++++++++--------------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/gst/compositor/compositororc-dist.c b/gst/compositor/compositororc-dist.c index 0fab506cd8..06cbfddead 100644 --- a/gst/compositor/compositororc-dist.c +++ b/gst/compositor/compositororc-dist.c @@ -123,11 +123,11 @@ void compositor_orc_overlay_bgra_addition (guint8 * ORC_RESTRICT d1, #define ORC_MAX(a,b) ((a)>(b) ? (a) : (b)) #define ORC_SB_MAX 127 #define ORC_SB_MIN (-1-ORC_SB_MAX) -#define ORC_UB_MAX 255 +#define ORC_UB_MAX (orc_uint8) 255 #define ORC_UB_MIN 0 #define ORC_SW_MAX 32767 #define ORC_SW_MIN (-1-ORC_SW_MAX) -#define ORC_UW_MAX 65535 +#define ORC_UW_MAX (orc_uint16)65535 #define ORC_UW_MIN 0 #define ORC_SL_MAX 2147483647 #define ORC_SL_MIN (-1-ORC_SL_MAX) @@ -601,7 +601,7 @@ compositor_orc_blend_argb (guint8 * ORC_RESTRICT d1, int d1_stride, var39.x4[2] = p1; var39.x4[3] = p1; /* 16: loadpl */ - var40.i = (int) 0x000000ff; /* 255 or 1.25987e-321f */ + var40.i = 0x000000ff; /* 255 or 1.25987e-321f */ for (i = 0; i < n; i++) { /* 0: loadl */ @@ -735,7 +735,7 @@ _backup_compositor_orc_blend_argb (OrcExecutor * ORC_RESTRICT ex) var39.x4[2] = ex->params[24]; var39.x4[3] = ex->params[24]; /* 16: loadpl */ - var40.i = (int) 0x000000ff; /* 255 or 1.25987e-321f */ + var40.i = 0x000000ff; /* 255 or 1.25987e-321f */ for (i = 0; i < n; i++) { /* 0: loadl */ @@ -975,7 +975,7 @@ compositor_orc_blend_bgra (guint8 * ORC_RESTRICT d1, int d1_stride, var40.x4[2] = p1; var40.x4[3] = p1; /* 17: loadpl */ - var41.i = (int) 0xff000000; /* -16777216 or 2.11371e-314f */ + var41.i = 0xff000000; /* -16777216 or 2.11371e-314f */ for (i = 0; i < n; i++) { /* 0: loadl */ @@ -1112,7 +1112,7 @@ _backup_compositor_orc_blend_bgra (OrcExecutor * ORC_RESTRICT ex) var40.x4[2] = ex->params[24]; var40.x4[3] = ex->params[24]; /* 17: loadpl */ - var41.i = (int) 0xff000000; /* -16777216 or 2.11371e-314f */ + var41.i = 0xff000000; /* -16777216 or 2.11371e-314f */ for (i = 0; i < n; i++) { /* 0: loadl */ @@ -1376,11 +1376,11 @@ compositor_orc_overlay_argb (guint8 * ORC_RESTRICT d1, int d1_stride, var41.x4[2] = p1; var41.x4[3] = p1; /* 10: loadpl */ - var53.i = (int) 0xffffffff; /* -1 or 2.122e-314f */ + var53.i = 0xffffffff; /* -1 or 2.122e-314f */ /* 26: loadpl */ - var42.i = (int) 0xffffff00; /* -256 or 2.122e-314f */ + var42.i = 0xffffff00; /* -256 or 2.122e-314f */ /* 29: loadpl */ - var43.i = (int) 0x000000ff; /* 255 or 1.25987e-321f */ + var43.i = 0x000000ff; /* 255 or 1.25987e-321f */ for (i = 0; i < n; i++) { /* 0: loadl */ @@ -1592,11 +1592,11 @@ _backup_compositor_orc_overlay_argb (OrcExecutor * ORC_RESTRICT ex) var41.x4[2] = ex->params[24]; var41.x4[3] = ex->params[24]; /* 10: loadpl */ - var53.i = (int) 0xffffffff; /* -1 or 2.122e-314f */ + var53.i = 0xffffffff; /* -1 or 2.122e-314f */ /* 26: loadpl */ - var42.i = (int) 0xffffff00; /* -256 or 2.122e-314f */ + var42.i = 0xffffff00; /* -256 or 2.122e-314f */ /* 29: loadpl */ - var43.i = (int) 0x000000ff; /* 255 or 1.25987e-321f */ + var43.i = 0x000000ff; /* 255 or 1.25987e-321f */ for (i = 0; i < n; i++) { /* 0: loadl */ @@ -1954,11 +1954,11 @@ compositor_orc_overlay_argb_addition (guint8 * ORC_RESTRICT d1, int d1_stride, var42.x4[2] = p1; var42.x4[3] = p1; /* 10: loadpl */ - var54.i = (int) 0xffffffff; /* -1 or 2.122e-314f */ + var54.i = 0xffffffff; /* -1 or 2.122e-314f */ /* 32: loadpl */ - var43.i = (int) 0xffffff00; /* -256 or 2.122e-314f */ + var43.i = 0xffffff00; /* -256 or 2.122e-314f */ /* 35: loadpl */ - var44.i = (int) 0x000000ff; /* 255 or 1.25987e-321f */ + var44.i = 0x000000ff; /* 255 or 1.25987e-321f */ for (i = 0; i < n; i++) { /* 0: loadl */ @@ -2197,11 +2197,11 @@ _backup_compositor_orc_overlay_argb_addition (OrcExecutor * ORC_RESTRICT ex) var42.x4[2] = ex->params[24]; var42.x4[3] = ex->params[24]; /* 10: loadpl */ - var54.i = (int) 0xffffffff; /* -1 or 2.122e-314f */ + var54.i = 0xffffffff; /* -1 or 2.122e-314f */ /* 32: loadpl */ - var43.i = (int) 0xffffff00; /* -256 or 2.122e-314f */ + var43.i = 0xffffff00; /* -256 or 2.122e-314f */ /* 35: loadpl */ - var44.i = (int) 0x000000ff; /* 255 or 1.25987e-321f */ + var44.i = 0x000000ff; /* 255 or 1.25987e-321f */ for (i = 0; i < n; i++) { /* 0: loadl */ @@ -2593,11 +2593,11 @@ compositor_orc_overlay_bgra (guint8 * ORC_RESTRICT d1, int d1_stride, var42.x4[2] = p1; var42.x4[3] = p1; /* 11: loadpl */ - var55.i = (int) 0xffffffff; /* -1 or 2.122e-314f */ + var55.i = 0xffffffff; /* -1 or 2.122e-314f */ /* 28: loadpl */ - var43.i = (int) 0x00ffffff; /* 16777215 or 8.28905e-317f */ + var43.i = 0x00ffffff; /* 16777215 or 8.28905e-317f */ /* 31: loadpl */ - var44.i = (int) 0xff000000; /* -16777216 or 2.11371e-314f */ + var44.i = 0xff000000; /* -16777216 or 2.11371e-314f */ for (i = 0; i < n; i++) { /* 0: loadl */ @@ -2815,11 +2815,11 @@ _backup_compositor_orc_overlay_bgra (OrcExecutor * ORC_RESTRICT ex) var42.x4[2] = ex->params[24]; var42.x4[3] = ex->params[24]; /* 11: loadpl */ - var55.i = (int) 0xffffffff; /* -1 or 2.122e-314f */ + var55.i = 0xffffffff; /* -1 or 2.122e-314f */ /* 28: loadpl */ - var43.i = (int) 0x00ffffff; /* 16777215 or 8.28905e-317f */ + var43.i = 0x00ffffff; /* 16777215 or 8.28905e-317f */ /* 31: loadpl */ - var44.i = (int) 0xff000000; /* -16777216 or 2.11371e-314f */ + var44.i = 0xff000000; /* -16777216 or 2.11371e-314f */ for (i = 0; i < n; i++) { /* 0: loadl */ @@ -3191,11 +3191,11 @@ compositor_orc_overlay_bgra_addition (guint8 * ORC_RESTRICT d1, int d1_stride, var43.x4[2] = p1; var43.x4[3] = p1; /* 11: loadpl */ - var56.i = (int) 0xffffffff; /* -1 or 2.122e-314f */ + var56.i = 0xffffffff; /* -1 or 2.122e-314f */ /* 35: loadpl */ - var44.i = (int) 0x00ffffff; /* 16777215 or 8.28905e-317f */ + var44.i = 0x00ffffff; /* 16777215 or 8.28905e-317f */ /* 38: loadpl */ - var45.i = (int) 0xff000000; /* -16777216 or 2.11371e-314f */ + var45.i = 0xff000000; /* -16777216 or 2.11371e-314f */ for (i = 0; i < n; i++) { /* 0: loadl */ @@ -3443,11 +3443,11 @@ _backup_compositor_orc_overlay_bgra_addition (OrcExecutor * ORC_RESTRICT ex) var43.x4[2] = ex->params[24]; var43.x4[3] = ex->params[24]; /* 11: loadpl */ - var56.i = (int) 0xffffffff; /* -1 or 2.122e-314f */ + var56.i = 0xffffffff; /* -1 or 2.122e-314f */ /* 35: loadpl */ - var44.i = (int) 0x00ffffff; /* 16777215 or 8.28905e-317f */ + var44.i = 0x00ffffff; /* 16777215 or 8.28905e-317f */ /* 38: loadpl */ - var45.i = (int) 0xff000000; /* -16777216 or 2.11371e-314f */ + var45.i = 0xff000000; /* -16777216 or 2.11371e-314f */ for (i = 0; i < n; i++) { /* 0: loadl */ From ea30f86ccc734c099edc21895d39f06b9ecc7c7f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim-Philipp=20M=C3=BCller?= Date: Thu, 15 Feb 2018 18:50:51 +0000 Subject: [PATCH 333/381] Dist compositor crossfade example and pythons script for meson build And add to autotools build so it gets disted. --- tests/examples/compositor/crossfade.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/examples/compositor/crossfade.c b/tests/examples/compositor/crossfade.c index ea2bb42ec2..44090ff556 100644 --- a/tests/examples/compositor/crossfade.c +++ b/tests/examples/compositor/crossfade.c @@ -37,7 +37,7 @@ typedef struct gboolean is_last; } VideoInfo; -gchar * +static gchar * ensure_uri (const gchar * location) { if (gst_uri_is_valid (location)) From b59da8eced6ff7305b36b4ca141aff71c138efe7 Mon Sep 17 00:00:00 2001 From: Mathieu Duponchelle Date: Thu, 1 Mar 2018 00:34:40 +0100 Subject: [PATCH 334/381] Port to latest GstAggregator segment API The aggregator segment is now exposed on the src pad https://bugzilla.gnome.org/show_bug.cgi?id=793946 --- ext/gl/gstglmixer.c | 3 +- ext/gl/gstglstereomix.c | 3 +- gst-libs/gst/video/gstvideoaggregator.c | 92 ++++++++++++++----------- gst/compositor/compositor.c | 3 +- 4 files changed, 56 insertions(+), 45 deletions(-) diff --git a/ext/gl/gstglmixer.c b/ext/gl/gstglmixer.c index 856a433f9d..2ea099ee03 100644 --- a/ext/gl/gstglmixer.c +++ b/ext/gl/gstglmixer.c @@ -364,7 +364,8 @@ gst_gl_mixer_class_init (GstGLMixerClass * klass) gobject_class->get_property = gst_gl_mixer_get_property; gobject_class->set_property = gst_gl_mixer_set_property; - gst_element_class_add_static_pad_template (element_class, &src_factory); + gst_element_class_add_static_pad_template_with_gtype (element_class, + &src_factory, GST_TYPE_AGGREGATOR_PAD); gst_element_class_add_static_pad_template_with_gtype (element_class, &sink_factory, GST_TYPE_GL_MIXER_PAD); diff --git a/ext/gl/gstglstereomix.c b/ext/gl/gstglstereomix.c index d3fe0302f2..42020a2b03 100644 --- a/ext/gl/gstglstereomix.c +++ b/ext/gl/gstglstereomix.c @@ -179,7 +179,8 @@ gst_gl_stereo_mix_class_init (GstGLStereoMixClass * klass) GST_TYPE_GL_STEREO_DOWNMIX_MODE_TYPE, DEFAULT_DOWNMIX, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); - gst_element_class_add_static_pad_template (element_class, &src_factory); + gst_element_class_add_static_pad_template_with_gtype (element_class, + &src_factory, GST_TYPE_AGGREGATOR_PAD); gst_element_class_add_static_pad_template_with_gtype (element_class, &sink_factory, GST_TYPE_GL_STEREO_MIX_PAD); diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index 6c043782ba..5a9dfc96ef 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -224,16 +224,17 @@ gst_video_aggregator_pad_skip_buffer (GstAggregatorPad * aggpad, GstAggregator * agg, GstBuffer * buffer) { gboolean ret = FALSE; + GstSegment *agg_segment = &GST_AGGREGATOR_PAD (agg->srcpad)->segment; - if (agg->segment.position != GST_CLOCK_TIME_NONE + if (agg_segment->position != GST_CLOCK_TIME_NONE && GST_BUFFER_DURATION (buffer) != GST_CLOCK_TIME_NONE) { GstClockTime start_time = - gst_segment_to_running_time (&aggpad->segment, GST_FORMAT_TIME, + gst_segment_to_running_time (agg_segment, GST_FORMAT_TIME, GST_BUFFER_PTS (buffer)); GstClockTime end_time = start_time + GST_BUFFER_DURATION (buffer); GstClockTime output_start_running_time = - gst_segment_to_running_time (&agg->segment, GST_FORMAT_TIME, - agg->segment.position); + gst_segment_to_running_time (agg_segment, GST_FORMAT_TIME, + agg_segment->position); ret = end_time < output_start_running_time; } @@ -744,7 +745,7 @@ gst_video_aggregator_default_negotiated_src_caps (GstAggregator * agg, if (GST_VIDEO_INFO_FPS_N (&vagg->info) != GST_VIDEO_INFO_FPS_N (&info) || GST_VIDEO_INFO_FPS_D (&vagg->info) != GST_VIDEO_INFO_FPS_D (&info)) { - if (agg->segment.position != -1) { + if (GST_AGGREGATOR_PAD (agg->srcpad)->segment.position != -1) { vagg->priv->nframes = 0; /* The timestamp offset will be updated based on the * segment position the next time we aggregate */ @@ -1054,7 +1055,7 @@ gst_video_aggregator_reset (GstVideoAggregator * vagg) vagg->priv->nframes = 0; vagg->priv->live = FALSE; - agg->segment.position = -1; + GST_AGGREGATOR_PAD (agg->srcpad)->segment.position = -1; gst_video_aggregator_reset_qos (vagg); @@ -1081,6 +1082,7 @@ gst_video_aggregator_fill_queues (GstVideoAggregator * vagg, gboolean eos = TRUE; gboolean need_more_data = FALSE; gboolean need_reconfigure = FALSE; + GstSegment *agg_segment = &GST_AGGREGATOR_PAD (agg->srcpad)->segment; /* get a set of buffers into pad->buffer that are within output_start_running_time * and output_end_running_time taking into account finished and unresponsive pads */ @@ -1189,9 +1191,9 @@ gst_video_aggregator_fill_queues (GstVideoAggregator * vagg, g_assert (start_time != -1 && end_time != -1); /* Convert to the output segment rate */ - if (ABS (agg->segment.rate) != 1.0) { - start_time *= ABS (agg->segment.rate); - end_time *= ABS (agg->segment.rate); + if (ABS (agg_segment->rate) != 1.0) { + start_time *= ABS (agg_segment->rate); + end_time *= ABS (agg_segment->rate); } GST_TRACE_OBJECT (pad, "dealing with buffer %p start %" GST_TIME_FORMAT @@ -1401,7 +1403,8 @@ gst_video_aggregator_do_qos (GstVideoAggregator * vagg, GstClockTime timestamp) /* qos is done on running time */ qostime = - gst_segment_to_running_time (&agg->segment, GST_FORMAT_TIME, timestamp); + gst_segment_to_running_time (&GST_AGGREGATOR_PAD (agg->srcpad)->segment, + GST_FORMAT_TIME, timestamp); /* see how our next timestamp relates to the latest qos timestamp */ GST_LOG_OBJECT (vagg, "qostime %" GST_TIME_FORMAT ", earliest %" @@ -1421,18 +1424,19 @@ static GstClockTime gst_video_aggregator_get_next_time (GstAggregator * agg) { GstClockTime next_time; + GstSegment *agg_segment = &GST_AGGREGATOR_PAD (agg->srcpad)->segment; GST_OBJECT_LOCK (agg); - if (agg->segment.position == -1 || agg->segment.position < agg->segment.start) - next_time = agg->segment.start; + if (agg_segment->position == -1 || agg_segment->position < agg_segment->start) + next_time = agg_segment->start; else - next_time = agg->segment.position; + next_time = agg_segment->position; - if (agg->segment.stop != -1 && next_time > agg->segment.stop) - next_time = agg->segment.stop; + if (agg_segment->stop != -1 && next_time > agg_segment->stop) + next_time = agg_segment->stop; next_time = - gst_segment_to_running_time (&agg->segment, GST_FORMAT_TIME, next_time); + gst_segment_to_running_time (agg_segment, GST_FORMAT_TIME, next_time); GST_OBJECT_UNLOCK (agg); return next_time; @@ -1444,13 +1448,14 @@ gst_video_aggregator_advance_on_timeout (GstVideoAggregator * vagg) GstAggregator *agg = GST_AGGREGATOR (vagg); guint64 frame_duration; gint fps_d, fps_n; + GstSegment *agg_segment = &GST_AGGREGATOR_PAD (agg->srcpad)->segment; GST_OBJECT_LOCK (agg); - if (agg->segment.position == -1) { - if (agg->segment.rate > 0.0) - agg->segment.position = agg->segment.start; + if (agg_segment->position == -1) { + if (agg_segment->rate > 0.0) + agg_segment->position = agg_segment->start; else - agg->segment.position = agg->segment.stop; + agg_segment->position = agg_segment->stop; } /* Advance position */ @@ -1460,12 +1465,12 @@ gst_video_aggregator_advance_on_timeout (GstVideoAggregator * vagg) GST_VIDEO_INFO_FPS_N (&vagg->info) : 25; /* Default to 25/1 if no "best fps" is known */ frame_duration = gst_util_uint64_scale (GST_SECOND, fps_d, fps_n); - if (agg->segment.rate > 0.0) - agg->segment.position += frame_duration; - else if (agg->segment.position > frame_duration) - agg->segment.position -= frame_duration; + if (agg_segment->rate > 0.0) + agg_segment->position += frame_duration; + else if (agg_segment->position > frame_duration) + agg_segment->position -= frame_duration; else - agg->segment.position = 0; + agg_segment->position = 0; vagg->priv->nframes++; GST_OBJECT_UNLOCK (agg); } @@ -1479,6 +1484,7 @@ gst_video_aggregator_aggregate (GstAggregator * agg, gboolean timeout) GstBuffer *outbuf = NULL; GstFlowReturn flow_ret; gint64 jitter; + GstSegment *agg_segment = &GST_AGGREGATOR_PAD (agg->srcpad)->segment; GST_VIDEO_AGGREGATOR_LOCK (vagg); @@ -1489,9 +1495,9 @@ gst_video_aggregator_aggregate (GstAggregator * agg, gboolean timeout) goto unlock_and_return; } - output_start_time = agg->segment.position; - if (agg->segment.position == -1 || agg->segment.position < agg->segment.start) - output_start_time = agg->segment.start; + output_start_time = agg_segment->position; + if (agg_segment->position == -1 || agg_segment->position < agg_segment->start) + output_start_time = agg_segment->start; if (vagg->priv->nframes == 0) { vagg->priv->ts_offset = output_start_time; @@ -1509,14 +1515,14 @@ gst_video_aggregator_aggregate (GstAggregator * agg, gboolean timeout) GST_VIDEO_INFO_FPS_N (&vagg->info)); } - if (agg->segment.stop != -1) - output_end_time = MIN (output_end_time, agg->segment.stop); + if (agg_segment->stop != -1) + output_end_time = MIN (output_end_time, agg_segment->stop); output_start_running_time = - gst_segment_to_running_time (&agg->segment, GST_FORMAT_TIME, + gst_segment_to_running_time (agg_segment, GST_FORMAT_TIME, output_start_time); output_end_running_time = - gst_segment_to_running_time (&agg->segment, GST_FORMAT_TIME, + gst_segment_to_running_time (agg_segment, GST_FORMAT_TIME, output_end_time); if (output_end_time == output_start_time) { @@ -1572,7 +1578,7 @@ gst_video_aggregator_aggregate (GstAggregator * agg, gboolean timeout) msg = gst_message_new_qos (GST_OBJECT_CAST (vagg), vagg->priv->live, - output_start_running_time, gst_segment_to_stream_time (&agg->segment, + output_start_running_time, gst_segment_to_stream_time (agg_segment, GST_FORMAT_TIME, output_start_time), output_start_time, output_end_time - output_start_time); gst_message_set_qos_values (msg, jitter, vagg->priv->proportion, 1000000); @@ -1595,7 +1601,7 @@ gst_video_aggregator_aggregate (GstAggregator * agg, gboolean timeout) GST_VIDEO_AGGREGATOR_LOCK (vagg); vagg->priv->nframes++; - agg->segment.position = output_end_time; + agg_segment->position = output_end_time; GST_VIDEO_AGGREGATOR_UNLOCK (vagg); return flow_ret; @@ -1701,6 +1707,7 @@ gst_video_aggregator_src_query (GstAggregator * agg, GstQuery * query) { GstVideoAggregator *vagg = GST_VIDEO_AGGREGATOR (agg); gboolean res = FALSE; + GstSegment *agg_segment = &GST_AGGREGATOR_PAD (agg->srcpad)->segment; switch (GST_QUERY_TYPE (query)) { case GST_QUERY_POSITION: @@ -1712,8 +1719,8 @@ gst_video_aggregator_src_query (GstAggregator * agg, GstQuery * query) switch (format) { case GST_FORMAT_TIME: gst_query_set_position (query, format, - gst_segment_to_stream_time (&agg->segment, GST_FORMAT_TIME, - agg->segment.position)); + gst_segment_to_stream_time (agg_segment, GST_FORMAT_TIME, + agg_segment->position)); res = TRUE; break; default: @@ -1778,18 +1785,19 @@ gst_video_aggregator_flush (GstAggregator * agg) GList *l; gdouble abs_rate; GstVideoAggregator *vagg = GST_VIDEO_AGGREGATOR (agg); + GstSegment *agg_segment = &GST_AGGREGATOR_PAD (agg->srcpad)->segment; GST_INFO_OBJECT (agg, "Flushing"); GST_OBJECT_LOCK (vagg); - abs_rate = ABS (agg->segment.rate); + abs_rate = ABS (agg_segment->rate); for (l = GST_ELEMENT (vagg)->sinkpads; l; l = l->next) { GstVideoAggregatorPad *p = l->data; /* Convert to the output segment rate */ - if (ABS (agg->segment.rate) != abs_rate) { - if (ABS (agg->segment.rate) != 1.0 && p->buffer) { - p->priv->start_time /= ABS (agg->segment.rate); - p->priv->end_time /= ABS (agg->segment.rate); + if (ABS (agg_segment->rate) != abs_rate) { + if (ABS (agg_segment->rate) != 1.0 && p->buffer) { + p->priv->start_time /= ABS (agg_segment->rate); + p->priv->end_time /= ABS (agg_segment->rate); } if (abs_rate != 1.0 && p->buffer) { p->priv->start_time *= abs_rate; @@ -1799,7 +1807,7 @@ gst_video_aggregator_flush (GstAggregator * agg) } GST_OBJECT_UNLOCK (vagg); - agg->segment.position = -1; + agg_segment->position = -1; vagg->priv->ts_offset = 0; vagg->priv->nframes = 0; diff --git a/gst/compositor/compositor.c b/gst/compositor/compositor.c index 2980e5c407..caa19f8e0e 100644 --- a/gst/compositor/compositor.c +++ b/gst/compositor/compositor.c @@ -1233,7 +1233,8 @@ gst_compositor_class_init (GstCompositorClass * klass) GST_TYPE_COMPOSITOR_BACKGROUND, DEFAULT_BACKGROUND, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); - gst_element_class_add_static_pad_template (gstelement_class, &src_factory); + gst_element_class_add_static_pad_template_with_gtype (gstelement_class, + &src_factory, GST_TYPE_AGGREGATOR_PAD); gst_element_class_add_static_pad_template_with_gtype (gstelement_class, &sink_factory, GST_TYPE_COMPOSITOR_PAD); From b69e7eef8ba10e90a8495c56b0b1ea6d5408f175 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim-Philipp=20M=C3=BCller?= Date: Tue, 13 Mar 2018 13:07:52 +0000 Subject: [PATCH 335/381] video: GST_EXPORT -> GST_URI_VIDEO_BAD_API We need different export decorators for the different libs. For now no actual change though, just rename before the release, and add prelude headers to define the new decorator to GST_EXPORT. --- gst-libs/gst/video/gstvideoaggregator.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/gst-libs/gst/video/gstvideoaggregator.h b/gst-libs/gst/video/gstvideoaggregator.h index 9882a75dfe..d3e5370b40 100644 --- a/gst-libs/gst/video/gstvideoaggregator.h +++ b/gst-libs/gst/video/gstvideoaggregator.h @@ -29,6 +29,7 @@ #include #include #include +#include G_BEGIN_DECLS @@ -111,7 +112,7 @@ struct _GstVideoAggregatorClass gpointer _gst_reserved[GST_PADDING_LARGE]; }; -GST_EXPORT +GST_VIDEO_BAD_API GType gst_video_aggregator_get_type (void); G_END_DECLS From 9ae9d2686f8c69de69c9d23993c3c2deee01d243 Mon Sep 17 00:00:00 2001 From: Xavier Claessens Date: Tue, 24 Apr 2018 14:05:30 -0400 Subject: [PATCH 336/381] Meson: Generate pc file for all plugins in bad https://bugzilla.gnome.org/show_bug.cgi?id=794568 --- gst/compositor/meson.build | 1 + 1 file changed, 1 insertion(+) diff --git a/gst/compositor/meson.build b/gst/compositor/meson.build index 52ca87385e..2af1b68e32 100644 --- a/gst/compositor/meson.build +++ b/gst/compositor/meson.build @@ -30,3 +30,4 @@ gstcompositor = library('gstcompositor', install : true, install_dir : plugins_install_dir, ) +pkgconfig.generate(gstcompositor, install_dir : plugins_pkgconfig_install_dir) From c13357f22ba77bc2f756297f4e797d2bb8e5cb84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Fri, 4 May 2018 14:53:21 +0200 Subject: [PATCH 337/381] videoaggregator: Get rid of separate header for the aggregator pad --- gst-libs/gst/video/gstvideoaggregator.c | 1 - gst-libs/gst/video/gstvideoaggregator.h | 89 +++++++++++++++++++++++-- 2 files changed, 83 insertions(+), 7 deletions(-) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index 5a9dfc96ef..93fa0cc1b6 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -42,7 +42,6 @@ #include #include "gstvideoaggregator.h" -#include "gstvideoaggregatorpad.h" GST_DEBUG_CATEGORY_STATIC (gst_video_aggregator_debug); #define GST_CAT_DEFAULT gst_video_aggregator_debug diff --git a/gst-libs/gst/video/gstvideoaggregator.h b/gst-libs/gst/video/gstvideoaggregator.h index d3e5370b40..633028aeae 100644 --- a/gst-libs/gst/video/gstvideoaggregator.h +++ b/gst-libs/gst/video/gstvideoaggregator.h @@ -33,6 +33,89 @@ G_BEGIN_DECLS +typedef struct _GstVideoAggregator GstVideoAggregator; +typedef struct _GstVideoAggregatorClass GstVideoAggregatorClass; +typedef struct _GstVideoAggregatorPrivate GstVideoAggregatorPrivate; + +#define GST_TYPE_VIDEO_AGGREGATOR_PAD (gst_video_aggregator_pad_get_type()) +#define GST_VIDEO_AGGREGATOR_PAD(obj) \ + (G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_VIDEO_AGGREGATOR_PAD, GstVideoAggregatorPad)) +#define GST_VIDEO_AGGREGATOR_PAD_CAST(obj) ((GstVideoAggregatorPad *)(obj)) +#define GST_VIDEO_AGGREGATOR_PAD_CLASS(klass) \ + (G_TYPE_CHECK_CLASS_CAST((klass),GST_TYPE_COMPOSITOR_PAD, GstVideoAggregatorPadClass)) +#define GST_IS_VIDEO_AGGREGATOR_PAD(obj) \ + (G_TYPE_CHECK_INSTANCE_TYPE((obj),GST_TYPE_VIDEO_AGGREGATOR_PAD)) +#define GST_IS_VIDEO_AGGREGATOR_PAD_CLASS(klass) \ + (G_TYPE_CHECK_CLASS_TYPE((klass),GST_TYPE_VIDEO_AGGREGATOR_PAD)) +#define GST_VIDEO_AGGREGATOR_PAD_GET_CLASS(obj) \ + (G_TYPE_INSTANCE_GET_CLASS((obj),GST_TYPE_VIDEO_AGGREGATOR_PAD,GstVideoAggregatorPadClass)) + +typedef struct _GstVideoAggregatorPad GstVideoAggregatorPad; +typedef struct _GstVideoAggregatorPadClass GstVideoAggregatorPadClass; +typedef struct _GstVideoAggregatorPadPrivate GstVideoAggregatorPadPrivate; + +/** + * GstVideoAggregatorPad: + * @info: The #GstVideoInfo currently set on the pad + * @buffer_vinfo: The #GstVideoInfo representing the type contained + * in @buffer + * @aggregated_frame: The #GstVideoFrame ready to be used for aggregation + * inside the aggregate_frames vmethod. + * @zorder: The zorder of this pad + */ +struct _GstVideoAggregatorPad +{ + GstAggregatorPad parent; + + GstVideoInfo info; + + GstBuffer *buffer; + + GstVideoFrame *aggregated_frame; + + /* properties */ + guint zorder; + gboolean ignore_eos; + + /* Subclasses can force an alpha channel in the (input thus output) + * colorspace format */ + gboolean needs_alpha; + + /* < private > */ + GstVideoAggregatorPadPrivate *priv; + + gpointer _gst_reserved[GST_PADDING]; +}; + +/** + * GstVideoAggregatorPadClass: + * + * @set_info: Lets subclass set a converter on the pad, + * right after a new format has been negotiated. + * @prepare_frame: Prepare the frame from the pad buffer (if any) + * and sets it to @aggregated_frame + * @clean_frame: clean the frame previously prepared in prepare_frame + */ +struct _GstVideoAggregatorPadClass +{ + GstAggregatorPadClass parent_class; + gboolean (*set_info) (GstVideoAggregatorPad * pad, + GstVideoAggregator * videoaggregator, + GstVideoInfo * current_info, + GstVideoInfo * wanted_info); + + gboolean (*prepare_frame) (GstVideoAggregatorPad * pad, + GstVideoAggregator * videoaggregator); + + void (*clean_frame) (GstVideoAggregatorPad * pad, + GstVideoAggregator * videoaggregator); + + gpointer _gst_reserved[GST_PADDING_LARGE]; +}; + +GST_VIDEO_BAD_API +GType gst_video_aggregator_pad_get_type (void); + #define GST_TYPE_VIDEO_AGGREGATOR (gst_video_aggregator_get_type()) #define GST_VIDEO_AGGREGATOR(obj) \ (G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_VIDEO_AGGREGATOR, GstVideoAggregator)) @@ -46,12 +129,6 @@ G_BEGIN_DECLS #define GST_VIDEO_AGGREGATOR_GET_CLASS(obj) \ (G_TYPE_INSTANCE_GET_CLASS((obj),GST_TYPE_VIDEO_AGGREGATOR,GstVideoAggregatorClass)) -typedef struct _GstVideoAggregator GstVideoAggregator; -typedef struct _GstVideoAggregatorClass GstVideoAggregatorClass; -typedef struct _GstVideoAggregatorPrivate GstVideoAggregatorPrivate; - -#include "gstvideoaggregatorpad.h" - /** * GstVideoAggregator: * @info: The #GstVideoInfo representing the currently set From a240fb7997480ba1606220d0f5fe2fc1c6b4e58f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Fri, 4 May 2018 16:13:16 +0200 Subject: [PATCH 338/381] videoaggregator: Move GstChildProxy implementations into leaf classes Not every subclass will want to expose the pads via the interface. https://bugzilla.gnome.org/show_bug.cgi?id=739011 --- ext/gl/gstglmosaic.c | 91 ++++++++++++++++++++++++ ext/gl/gstglstereomix.c | 92 ++++++++++++++++++++++++- ext/gl/gstglvideomixer.c | 74 ++++++++++++++++++++ gst-libs/gst/video/gstvideoaggregator.c | 55 --------------- gst/compositor/compositor.c | 91 +++++++++++++++++++++++- 5 files changed, 346 insertions(+), 57 deletions(-) diff --git a/ext/gl/gstglmosaic.c b/ext/gl/gstglmosaic.c index 77aec599cd..63075ba5a9 100644 --- a/ext/gl/gstglmosaic.c +++ b/ext/gl/gstglmosaic.c @@ -55,10 +55,15 @@ enum PROP_0, }; +static void gst_gl_mosaic_child_proxy_init (gpointer g_iface, + gpointer iface_data); + #define DEBUG_INIT \ GST_DEBUG_CATEGORY_INIT (gst_gl_mosaic_debug, "glmosaic", 0, "glmosaic element"); G_DEFINE_TYPE_WITH_CODE (GstGLMosaic, gst_gl_mosaic, GST_TYPE_GL_MIXER, + G_IMPLEMENT_INTERFACE (GST_TYPE_CHILD_PROXY, + gst_gl_mosaic_child_proxy_init); DEBUG_INIT); static void gst_gl_mosaic_set_property (GObject * object, guint prop_id, @@ -66,6 +71,10 @@ static void gst_gl_mosaic_set_property (GObject * object, guint prop_id, static void gst_gl_mosaic_get_property (GObject * object, guint prop_id, GValue * value, GParamSpec * pspec); +static GstPad *gst_gl_mosaic_request_new_pad (GstElement * element, + GstPadTemplate * temp, const gchar * req_name, const GstCaps * caps); +static void gst_gl_mosaic_release_pad (GstElement * element, GstPad * pad); + static void gst_gl_mosaic_reset (GstGLMixer * mixer); static gboolean gst_gl_mosaic_init_shader (GstGLMixer * mixer, GstCaps * outcaps); @@ -128,6 +137,10 @@ gst_gl_mosaic_class_init (GstGLMosaicClass * klass) gobject_class->set_property = gst_gl_mosaic_set_property; gobject_class->get_property = gst_gl_mosaic_get_property; + element_class->request_new_pad = + GST_DEBUG_FUNCPTR (gst_gl_mosaic_request_new_pad); + element_class->release_pad = GST_DEBUG_FUNCPTR (gst_gl_mosaic_release_pad); + gst_element_class_set_metadata (element_class, "OpenGL mosaic", "Filter/Effect/Video", "OpenGL mosaic", "Julien Isorce "); @@ -171,6 +184,44 @@ gst_gl_mosaic_get_property (GObject * object, guint prop_id, } } +static GstPad * +gst_gl_mosaic_request_new_pad (GstElement * element, GstPadTemplate * templ, + const gchar * req_name, const GstCaps * caps) +{ + GstPad *newpad; + + newpad = (GstPad *) + GST_ELEMENT_CLASS (gst_gl_mosaic_parent_class)->request_new_pad (element, + templ, req_name, caps); + + if (newpad == NULL) + goto could_not_create; + + gst_child_proxy_child_added (GST_CHILD_PROXY (element), G_OBJECT (newpad), + GST_OBJECT_NAME (newpad)); + + return newpad; + +could_not_create: + { + GST_DEBUG_OBJECT (element, "could not create/add pad"); + return NULL; + } +} + +static void +gst_gl_mosaic_release_pad (GstElement * element, GstPad * pad) +{ + GstGLMosaic *gl_mosaic = GST_GL_MOSAIC (element); + + GST_DEBUG_OBJECT (gl_mosaic, "release pad %s:%s", GST_DEBUG_PAD_NAME (pad)); + + gst_child_proxy_child_removed (GST_CHILD_PROXY (gl_mosaic), G_OBJECT (pad), + GST_OBJECT_NAME (pad)); + + GST_ELEMENT_CLASS (gst_gl_mosaic_parent_class)->release_pad (element, pad); +} + static void gst_gl_mosaic_reset (GstGLMixer * mixer) { @@ -356,3 +407,43 @@ gst_gl_mosaic_callback (gpointer stuff) return TRUE; } + +/* GstChildProxy implementation */ +static GObject * +gst_gl_mosaic_child_proxy_get_child_by_index (GstChildProxy * child_proxy, + guint index) +{ + GstGLMosaic *gl_mosaic = GST_GL_MOSAIC (child_proxy); + GObject *obj = NULL; + + GST_OBJECT_LOCK (gl_mosaic); + obj = g_list_nth_data (GST_ELEMENT_CAST (gl_mosaic)->sinkpads, index); + if (obj) + gst_object_ref (obj); + GST_OBJECT_UNLOCK (gl_mosaic); + + return obj; +} + +static guint +gst_gl_mosaic_child_proxy_get_children_count (GstChildProxy * child_proxy) +{ + guint count = 0; + GstGLMosaic *gl_mosaic = GST_GL_MOSAIC (child_proxy); + + GST_OBJECT_LOCK (gl_mosaic); + count = GST_ELEMENT_CAST (gl_mosaic)->numsinkpads; + GST_OBJECT_UNLOCK (gl_mosaic); + GST_INFO_OBJECT (gl_mosaic, "Children Count: %d", count); + + return count; +} + +static void +gst_gl_mosaic_child_proxy_init (gpointer g_iface, gpointer iface_data) +{ + GstChildProxyInterface *iface = g_iface; + + iface->get_child_by_index = gst_gl_mosaic_child_proxy_get_child_by_index; + iface->get_children_count = gst_gl_mosaic_child_proxy_get_children_count; +} diff --git a/ext/gl/gstglstereomix.c b/ext/gl/gstglstereomix.c index 42020a2b03..4d72dfa3dd 100644 --- a/ext/gl/gstglstereomix.c +++ b/ext/gl/gstglstereomix.c @@ -78,8 +78,13 @@ gst_gl_stereo_mix_pad_init (GstGLStereoMixPad * pad) { } +static void gst_gl_stereo_mix_child_proxy_init (gpointer g_iface, + gpointer iface_data); + #define gst_gl_stereo_mix_parent_class parent_class -G_DEFINE_TYPE (GstGLStereoMix, gst_gl_stereo_mix, GST_TYPE_GL_MIXER); +G_DEFINE_TYPE_WITH_CODE (GstGLStereoMix, gst_gl_stereo_mix, GST_TYPE_GL_MIXER, + G_IMPLEMENT_INTERFACE (GST_TYPE_CHILD_PROXY, + gst_gl_stereo_mix_child_proxy_init)); static GstCaps *_update_caps (GstVideoAggregator * vagg, GstCaps * caps); static gboolean _negotiated_caps (GstAggregator * aggregator, GstCaps * caps); @@ -133,6 +138,10 @@ static GstStaticPadTemplate sink_factory = GST_STATIC_PAD_TEMPLATE ("sink_%u", "; " GST_VIDEO_CAPS_MAKE (GST_GL_COLOR_CONVERT_FORMATS)) ); +static GstPad *gst_gl_stereo_mix_request_new_pad (GstElement * element, + GstPadTemplate * temp, const gchar * req_name, const GstCaps * caps); +static void gst_gl_stereo_mix_release_pad (GstElement * element, GstPad * pad); + static GstFlowReturn gst_gl_stereo_mix_get_output_buffer (GstVideoAggregator * videoaggregator, GstBuffer ** outbuf); static gboolean gst_gl_stereo_mix_stop (GstAggregator * agg); @@ -184,6 +193,11 @@ gst_gl_stereo_mix_class_init (GstGLStereoMixClass * klass) gst_element_class_add_static_pad_template_with_gtype (element_class, &sink_factory, GST_TYPE_GL_STEREO_MIX_PAD); + element_class->request_new_pad = + GST_DEBUG_FUNCPTR (gst_gl_stereo_mix_request_new_pad); + element_class->release_pad = + GST_DEBUG_FUNCPTR (gst_gl_stereo_mix_release_pad); + agg_class->stop = gst_gl_stereo_mix_stop; agg_class->start = gst_gl_stereo_mix_start; agg_class->src_query = gst_gl_stereo_mix_src_query; @@ -399,6 +413,42 @@ gst_gl_stereo_mix_set_property (GObject * object, } } +static GstPad * +gst_gl_stereo_mix_request_new_pad (GstElement * element, GstPadTemplate * templ, + const gchar * req_name, const GstCaps * caps) +{ + GstPad *newpad; + + newpad = (GstPad *) + GST_ELEMENT_CLASS (parent_class)->request_new_pad (element, + templ, req_name, caps); + + if (newpad == NULL) + goto could_not_create; + + gst_child_proxy_child_added (GST_CHILD_PROXY (element), G_OBJECT (newpad), + GST_OBJECT_NAME (newpad)); + + return GST_PAD_CAST (newpad); + +could_not_create: + { + GST_DEBUG_OBJECT (element, "could not create/add pad"); + return NULL; + } +} + +static void +gst_gl_stereo_mix_release_pad (GstElement * element, GstPad * pad) +{ + GST_DEBUG_OBJECT (element, "release pad %s:%s", GST_DEBUG_PAD_NAME (pad)); + + gst_child_proxy_child_removed (GST_CHILD_PROXY (element), G_OBJECT (pad), + GST_OBJECT_NAME (pad)); + + GST_ELEMENT_CLASS (parent_class)->release_pad (element, pad); +} + static gboolean gst_gl_stereo_mix_start (GstAggregator * agg) { @@ -696,3 +746,43 @@ gst_gl_stereo_mix_process_frames (GstGLStereoMix * mixer) return TRUE; } + +/* GstChildProxy implementation */ +static GObject * +gst_gl_stereo_mix_child_proxy_get_child_by_index (GstChildProxy * child_proxy, + guint index) +{ + GstGLStereoMix *gl_stereo_mix = GST_GL_STEREO_MIX (child_proxy); + GObject *obj = NULL; + + GST_OBJECT_LOCK (gl_stereo_mix); + obj = g_list_nth_data (GST_ELEMENT_CAST (gl_stereo_mix)->sinkpads, index); + if (obj) + gst_object_ref (obj); + GST_OBJECT_UNLOCK (gl_stereo_mix); + + return obj; +} + +static guint +gst_gl_stereo_mix_child_proxy_get_children_count (GstChildProxy * child_proxy) +{ + guint count = 0; + GstGLStereoMix *gl_stereo_mix = GST_GL_STEREO_MIX (child_proxy); + + GST_OBJECT_LOCK (gl_stereo_mix); + count = GST_ELEMENT_CAST (gl_stereo_mix)->numsinkpads; + GST_OBJECT_UNLOCK (gl_stereo_mix); + GST_INFO_OBJECT (gl_stereo_mix, "Children Count: %d", count); + + return count; +} + +static void +gst_gl_stereo_mix_child_proxy_init (gpointer g_iface, gpointer iface_data) +{ + GstChildProxyInterface *iface = g_iface; + + iface->get_child_by_index = gst_gl_stereo_mix_child_proxy_get_child_by_index; + iface->get_children_count = gst_gl_stereo_mix_child_proxy_get_children_count; +} diff --git a/ext/gl/gstglvideomixer.c b/ext/gl/gstglvideomixer.c index 78b5d06c51..602aace373 100644 --- a/ext/gl/gstglvideomixer.c +++ b/ext/gl/gstglvideomixer.c @@ -451,11 +451,16 @@ enum PROP_BACKGROUND, }; +static void gst_gl_video_mixer_child_proxy_init (gpointer g_iface, + gpointer iface_data); + #define DEBUG_INIT \ GST_DEBUG_CATEGORY_INIT (gst_gl_video_mixer_debug, "glvideomixer", 0, "glvideomixer element"); #define gst_gl_video_mixer_parent_class parent_class G_DEFINE_TYPE_WITH_CODE (GstGLVideoMixer, gst_gl_video_mixer, GST_TYPE_GL_MIXER, + G_IMPLEMENT_INTERFACE (GST_TYPE_CHILD_PROXY, + gst_gl_video_mixer_child_proxy_init); DEBUG_INIT); static void gst_gl_video_mixer_set_property (GObject * object, guint prop_id, @@ -837,11 +842,39 @@ _del_buffer (GstGLContext * context, GLuint * pBuffer) context->gl_vtable->DeleteBuffers (1, pBuffer); } +static GstPad * +gst_gl_video_mixer_request_new_pad (GstElement * element, + GstPadTemplate * templ, const gchar * req_name, const GstCaps * caps) +{ + GstPad *newpad; + + newpad = (GstPad *) + GST_ELEMENT_CLASS (parent_class)->request_new_pad (element, + templ, req_name, caps); + + if (newpad == NULL) + goto could_not_create; + + gst_child_proxy_child_added (GST_CHILD_PROXY (element), G_OBJECT (newpad), + GST_OBJECT_NAME (newpad)); + + return newpad; + +could_not_create: + { + GST_DEBUG_OBJECT (element, "could not create/add pad"); + return NULL; + } +} + static void gst_gl_video_mixer_release_pad (GstElement * element, GstPad * p) { GstGLVideoMixerPad *pad = GST_GL_VIDEO_MIXER_PAD (p); + gst_child_proxy_child_removed (GST_CHILD_PROXY (element), G_OBJECT (pad), + GST_OBJECT_NAME (pad)); + /* we call the base class first as this will remove the pad from * the aggregator, thus stopping misc callbacks from being called, * one of which (process_textures) will recreate the vertex_buffer @@ -867,6 +900,7 @@ gst_gl_video_mixer_class_init (GstGLVideoMixerClass * klass) gobject_class = (GObjectClass *) klass; element_class = GST_ELEMENT_CLASS (klass); + element_class->request_new_pad = gst_gl_video_mixer_request_new_pad; element_class->release_pad = gst_gl_video_mixer_release_pad; gobject_class->set_property = gst_gl_video_mixer_set_property; @@ -1590,3 +1624,43 @@ gst_gl_video_mixer_callback (gpointer stuff) return TRUE; } + +/* GstChildProxy implementation */ +static GObject * +gst_gl_video_mixer_child_proxy_get_child_by_index (GstChildProxy * child_proxy, + guint index) +{ + GstGLVideoMixer *gl_video_mixer = GST_GL_VIDEO_MIXER (child_proxy); + GObject *obj = NULL; + + GST_OBJECT_LOCK (gl_video_mixer); + obj = g_list_nth_data (GST_ELEMENT_CAST (gl_video_mixer)->sinkpads, index); + if (obj) + gst_object_ref (obj); + GST_OBJECT_UNLOCK (gl_video_mixer); + + return obj; +} + +static guint +gst_gl_video_mixer_child_proxy_get_children_count (GstChildProxy * child_proxy) +{ + guint count = 0; + GstGLVideoMixer *gl_video_mixer = GST_GL_VIDEO_MIXER (child_proxy); + + GST_OBJECT_LOCK (gl_video_mixer); + count = GST_ELEMENT_CAST (gl_video_mixer)->numsinkpads; + GST_OBJECT_UNLOCK (gl_video_mixer); + GST_INFO_OBJECT (gl_video_mixer, "Children Count: %d", count); + + return count; +} + +static void +gst_gl_video_mixer_child_proxy_init (gpointer g_iface, gpointer iface_data) +{ + GstChildProxyInterface *iface = g_iface; + + iface->get_child_by_index = gst_gl_video_mixer_child_proxy_get_child_by_index; + iface->get_children_count = gst_gl_video_mixer_child_proxy_get_children_count; +} diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index 93fa0cc1b6..f5556fe6a1 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -369,51 +369,6 @@ gst_video_aggregator_pad_init (GstVideoAggregatorPad * vaggpad) vaggpad->priv->convert = NULL; } -/********************************* - * GstChildProxy implementation * - *********************************/ -static GObject * -gst_video_aggregator_child_proxy_get_child_by_index (GstChildProxy * - child_proxy, guint index) -{ - GObject *obj; - GstVideoAggregator *vagg = GST_VIDEO_AGGREGATOR (child_proxy); - - GST_OBJECT_LOCK (vagg); - if ((obj = g_list_nth_data (GST_ELEMENT (vagg)->sinkpads, index))) - g_object_ref (obj); - GST_OBJECT_UNLOCK (vagg); - - return obj; -} - -static guint -gst_video_aggregator_child_proxy_get_children_count (GstChildProxy * - child_proxy) -{ - guint count = 0; - - GST_OBJECT_LOCK (child_proxy); - count = GST_ELEMENT (child_proxy)->numsinkpads; - GST_OBJECT_UNLOCK (child_proxy); - - GST_INFO_OBJECT (child_proxy, "Children Count: %d", count); - - return count; -} - -static void -gst_video_aggregator_child_proxy_init (gpointer g_iface, gpointer iface_data) -{ - GstChildProxyInterface *iface = g_iface; - - GST_INFO ("intializing child proxy interface"); - iface->get_child_by_index = - gst_video_aggregator_child_proxy_get_child_by_index; - iface->get_children_count = - gst_video_aggregator_child_proxy_get_children_count; -} - /************************************** * GstVideoAggregator implementation * **************************************/ @@ -479,10 +434,6 @@ gst_video_aggregator_get_type (void) sizeof (GstVideoAggregator), (GInstanceInitFunc) gst_video_aggregator_init, (GTypeFlags) G_TYPE_FLAG_ABSTRACT); - { - G_IMPLEMENT_INTERFACE (GST_TYPE_CHILD_PROXY, - gst_video_aggregator_child_proxy_init); - } g_once_init_leave (&g_define_type_id_volatile, g_define_type_id); } return g_define_type_id_volatile; @@ -1902,9 +1853,6 @@ gst_video_aggregator_request_new_pad (GstElement * element, (GCompareFunc) pad_zorder_compare); GST_OBJECT_UNLOCK (vagg); - gst_child_proxy_child_added (GST_CHILD_PROXY (vagg), G_OBJECT (vaggpad), - GST_OBJECT_NAME (vaggpad)); - return GST_PAD (vaggpad); } @@ -1929,9 +1877,6 @@ gst_video_aggregator_release_pad (GstElement * element, GstPad * pad) gst_buffer_replace (&vaggpad->buffer, NULL); - gst_child_proxy_child_removed (GST_CHILD_PROXY (vagg), G_OBJECT (vaggpad), - GST_OBJECT_NAME (vaggpad)); - GST_ELEMENT_CLASS (gst_video_aggregator_parent_class)->release_pad (GST_ELEMENT (vagg), pad); diff --git a/gst/compositor/compositor.c b/gst/compositor/compositor.c index caa19f8e0e..66cf36ccf5 100644 --- a/gst/compositor/compositor.c +++ b/gst/compositor/compositor.c @@ -120,6 +120,9 @@ static GstStaticPadTemplate sink_factory = GST_STATIC_PAD_TEMPLATE ("sink_%u", GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE (FORMATS)) ); +static void gst_compositor_child_proxy_init (gpointer g_iface, + gpointer iface_data); + #define DEFAULT_PAD_XPOS 0 #define DEFAULT_PAD_YPOS 0 #define DEFAULT_PAD_WIDTH 0 @@ -731,7 +734,9 @@ gst_compositor_set_property (GObject * object, } #define gst_compositor_parent_class parent_class -G_DEFINE_TYPE (GstCompositor, gst_compositor, GST_TYPE_VIDEO_AGGREGATOR); +G_DEFINE_TYPE_WITH_CODE (GstCompositor, gst_compositor, + GST_TYPE_VIDEO_AGGREGATOR, G_IMPLEMENT_INTERFACE (GST_TYPE_CHILD_PROXY, + gst_compositor_child_proxy_init)); static gboolean set_functions (GstCompositor * self, GstVideoInfo * info) @@ -1168,6 +1173,46 @@ gst_compositor_aggregate_frames (GstVideoAggregator * vagg, GstBuffer * outbuf) return GST_FLOW_OK; } +static GstPad * +gst_compositor_request_new_pad (GstElement * element, GstPadTemplate * templ, + const gchar * req_name, const GstCaps * caps) +{ + GstPad *newpad; + + newpad = (GstPad *) + GST_ELEMENT_CLASS (parent_class)->request_new_pad (element, + templ, req_name, caps); + + if (newpad == NULL) + goto could_not_create; + + gst_child_proxy_child_added (GST_CHILD_PROXY (element), G_OBJECT (newpad), + GST_OBJECT_NAME (newpad)); + + return newpad; + +could_not_create: + { + GST_DEBUG_OBJECT (element, "could not create/add pad"); + return NULL; + } +} + +static void +gst_compositor_release_pad (GstElement * element, GstPad * pad) +{ + GstCompositor *compositor; + + compositor = GST_COMPOSITOR (element); + + GST_DEBUG_OBJECT (compositor, "release pad %s:%s", GST_DEBUG_PAD_NAME (pad)); + + gst_child_proxy_child_removed (GST_CHILD_PROXY (compositor), G_OBJECT (pad), + GST_OBJECT_NAME (pad)); + + GST_ELEMENT_CLASS (parent_class)->release_pad (element, pad); +} + static gboolean _sink_query (GstAggregator * agg, GstAggregatorPad * bpad, GstQuery * query) { @@ -1223,6 +1268,10 @@ gst_compositor_class_init (GstCompositorClass * klass) gobject_class->get_property = gst_compositor_get_property; gobject_class->set_property = gst_compositor_set_property; + gstelement_class->request_new_pad = + GST_DEBUG_FUNCPTR (gst_compositor_request_new_pad); + gstelement_class->release_pad = + GST_DEBUG_FUNCPTR (gst_compositor_release_pad); agg_class->sink_query = _sink_query; agg_class->fixate_src_caps = _fixate_caps; agg_class->negotiated_src_caps = _negotiated_caps; @@ -1251,6 +1300,46 @@ gst_compositor_init (GstCompositor * self) self->background = DEFAULT_BACKGROUND; } +/* GstChildProxy implementation */ +static GObject * +gst_compositor_child_proxy_get_child_by_index (GstChildProxy * child_proxy, + guint index) +{ + GstCompositor *compositor = GST_COMPOSITOR (child_proxy); + GObject *obj = NULL; + + GST_OBJECT_LOCK (compositor); + obj = g_list_nth_data (GST_ELEMENT_CAST (compositor)->sinkpads, index); + if (obj) + gst_object_ref (obj); + GST_OBJECT_UNLOCK (compositor); + + return obj; +} + +static guint +gst_compositor_child_proxy_get_children_count (GstChildProxy * child_proxy) +{ + guint count = 0; + GstCompositor *compositor = GST_COMPOSITOR (child_proxy); + + GST_OBJECT_LOCK (compositor); + count = GST_ELEMENT_CAST (compositor)->numsinkpads; + GST_OBJECT_UNLOCK (compositor); + GST_INFO_OBJECT (compositor, "Children Count: %d", count); + + return count; +} + +static void +gst_compositor_child_proxy_init (gpointer g_iface, gpointer iface_data) +{ + GstChildProxyInterface *iface = g_iface; + + iface->get_child_by_index = gst_compositor_child_proxy_get_child_by_index; + iface->get_children_count = gst_compositor_child_proxy_get_children_count; +} + /* Element registration */ static gboolean plugin_init (GstPlugin * plugin) From 51c359dda68cc09abc5161fe2f7167a2a800a61d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Fri, 4 May 2018 16:46:00 +0200 Subject: [PATCH 339/381] videoaggregator: Rename ignore-eos pad property to repeat-after-eos What it does is to repeat the last frame forever after EOS, it does not literally ignore EOS. --- ext/gl/gstglvideomixer.c | 22 +++++++++++----------- gst-libs/gst/video/gstvideoaggregator.c | 24 ++++++++++++------------ gst-libs/gst/video/gstvideoaggregator.h | 2 +- tests/check/elements/compositor.c | 8 ++++---- 4 files changed, 28 insertions(+), 28 deletions(-) diff --git a/ext/gl/gstglvideomixer.c b/ext/gl/gstglvideomixer.c index 602aace373..26a42d62de 100644 --- a/ext/gl/gstglvideomixer.c +++ b/ext/gl/gstglvideomixer.c @@ -157,7 +157,7 @@ gst_gl_video_mixer_blend_function_get_type (void) #define DEFAULT_PAD_HEIGHT 0 #define DEFAULT_PAD_ALPHA 1.0 #define DEFAULT_PAD_ZORDER 0 -#define DEFAULT_PAD_IGNORE_EOS FALSE +#define DEFAULT_PAD_REPEAT_AFTER_EOS FALSE #define DEFAULT_PAD_BLEND_EQUATION_RGB GST_GL_VIDEO_MIXER_BLEND_EQUATION_ADD #define DEFAULT_PAD_BLEND_EQUATION_ALPHA GST_GL_VIDEO_MIXER_BLEND_EQUATION_ADD #define DEFAULT_PAD_BLEND_FUNCTION_SRC_RGB GST_GL_VIDEO_MIXER_BLEND_FUNCTION_SRC_ALPHA @@ -184,7 +184,7 @@ enum PROP_INPUT_BLEND_FUNCTION_CONSTANT_COLOR_BLUE, PROP_INPUT_BLEND_FUNCTION_CONSTANT_COLOR_ALPHA, PROP_INPUT_ZORDER, - PROP_INPUT_IGNORE_EOS, + PROP_INPUT_REPEAT_AFTER_EOS, }; static void gst_gl_video_mixer_input_get_property (GObject * object, @@ -225,10 +225,11 @@ gst_gl_video_mixer_input_class_init (GstGLVideoMixerInputClass * klass) g_param_spec_uint ("zorder", "Z-Order", "Z Order of the picture", 0, 10000, DEFAULT_PAD_ZORDER, G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); - g_object_class_install_property (gobject_class, PROP_INPUT_IGNORE_EOS, - g_param_spec_boolean ("ignore-eos", "Ignore EOS", "Aggregate the last " + g_object_class_install_property (gobject_class, PROP_INPUT_REPEAT_AFTER_EOS, + g_param_spec_boolean ("repeat-after-eos", "Repeat After EOS", + "Aggregate the last " "frame on pads that are EOS till they are released", - DEFAULT_PAD_IGNORE_EOS, + DEFAULT_PAD_REPEAT_AFTER_EOS, G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); g_object_class_install_property (gobject_class, PROP_INPUT_XPOS, g_param_spec_int ("xpos", "X Position", "X Position of the picture", @@ -239,12 +240,12 @@ gst_gl_video_mixer_input_class_init (GstGLVideoMixerInputClass * klass) G_MININT, G_MAXINT, DEFAULT_PAD_YPOS, G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); g_object_class_install_property (gobject_class, PROP_INPUT_WIDTH, - g_param_spec_int ("width", "Width", "Width of the picture", - G_MININT, G_MAXINT, DEFAULT_PAD_WIDTH, + g_param_spec_int ("width", "Width", "Width of the picture", G_MININT, + G_MAXINT, DEFAULT_PAD_WIDTH, G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); g_object_class_install_property (gobject_class, PROP_INPUT_HEIGHT, - g_param_spec_int ("height", "Height", "Height of the picture", - G_MININT, G_MAXINT, DEFAULT_PAD_HEIGHT, + g_param_spec_int ("height", "Height", "Height of the picture", G_MININT, + G_MAXINT, DEFAULT_PAD_HEIGHT, G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); g_object_class_install_property (gobject_class, PROP_INPUT_ALPHA, g_param_spec_double ("alpha", "Alpha", "Alpha of the picture", 0.0, 1.0, @@ -252,8 +253,7 @@ gst_gl_video_mixer_input_class_init (GstGLVideoMixerInputClass * klass) G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); g_object_class_install_property (gobject_class, PROP_INPUT_BLEND_EQUATION_RGB, g_param_spec_enum ("blend-equation-rgb", "Blend Equation RGB", - "Blend Equation for RGB", - GST_TYPE_GL_VIDEO_MIXER_BLEND_EQUATION, + "Blend Equation for RGB", GST_TYPE_GL_VIDEO_MIXER_BLEND_EQUATION, DEFAULT_PAD_BLEND_EQUATION_RGB, G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); g_object_class_install_property (gobject_class, diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index f5556fe6a1..29251a0048 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -54,12 +54,12 @@ static void gst_video_aggregator_reset_qos (GstVideoAggregator * vagg); ****************************************/ #define DEFAULT_PAD_ZORDER 0 -#define DEFAULT_PAD_IGNORE_EOS FALSE +#define DEFAULT_PAD_REPEAT_AFTER_EOS FALSE enum { PROP_PAD_0, PROP_PAD_ZORDER, - PROP_PAD_IGNORE_EOS, + PROP_PAD_REPEAT_AFTER_EOS, }; @@ -92,8 +92,8 @@ gst_video_aggregator_pad_get_property (GObject * object, guint prop_id, case PROP_PAD_ZORDER: g_value_set_uint (value, pad->zorder); break; - case PROP_PAD_IGNORE_EOS: - g_value_set_boolean (value, pad->ignore_eos); + case PROP_PAD_REPEAT_AFTER_EOS: + g_value_set_boolean (value, pad->repeat_after_eos); break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); @@ -124,8 +124,8 @@ gst_video_aggregator_pad_set_property (GObject * object, guint prop_id, (GCompareFunc) pad_zorder_compare); GST_OBJECT_UNLOCK (vagg); break; - case PROP_PAD_IGNORE_EOS: - pad->ignore_eos = g_value_get_boolean (value); + case PROP_PAD_REPEAT_AFTER_EOS: + pad->repeat_after_eos = g_value_get_boolean (value); break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); @@ -337,10 +337,10 @@ gst_video_aggregator_pad_class_init (GstVideoAggregatorPadClass * klass) g_param_spec_uint ("zorder", "Z-Order", "Z Order of the picture", 0, G_MAXUINT, DEFAULT_PAD_ZORDER, G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); - g_object_class_install_property (gobject_class, PROP_PAD_IGNORE_EOS, - g_param_spec_boolean ("ignore-eos", "Ignore EOS", "Aggregate the last " - "frame on pads that are EOS till they are released", - DEFAULT_PAD_IGNORE_EOS, + g_object_class_install_property (gobject_class, PROP_PAD_REPEAT_AFTER_EOS, + g_param_spec_boolean ("repeat-after-eos", "Repeat After EOS", + "Repeat the " "last frame after EOS until all pads are EOS", + DEFAULT_PAD_REPEAT_AFTER_EOS, G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); g_type_class_add_private (klass, sizeof (GstVideoAggregatorPadPrivate)); @@ -362,7 +362,7 @@ gst_video_aggregator_pad_init (GstVideoAggregatorPad * vaggpad) GstVideoAggregatorPadPrivate); vaggpad->zorder = DEFAULT_PAD_ZORDER; - vaggpad->ignore_eos = DEFAULT_PAD_IGNORE_EOS; + vaggpad->repeat_after_eos = DEFAULT_PAD_REPEAT_AFTER_EOS; vaggpad->aggregated_frame = NULL; vaggpad->priv->converted_buffer = NULL; @@ -1201,7 +1201,7 @@ gst_video_aggregator_fill_queues (GstVideoAggregator * vagg, continue; } } else { - if (is_eos && pad->ignore_eos) { + if (is_eos && pad->repeat_after_eos) { eos = FALSE; GST_DEBUG_OBJECT (pad, "ignoring EOS and re-using previous buffer"); continue; diff --git a/gst-libs/gst/video/gstvideoaggregator.h b/gst-libs/gst/video/gstvideoaggregator.h index 633028aeae..a1b7f8d9c2 100644 --- a/gst-libs/gst/video/gstvideoaggregator.h +++ b/gst-libs/gst/video/gstvideoaggregator.h @@ -75,7 +75,7 @@ struct _GstVideoAggregatorPad /* properties */ guint zorder; - gboolean ignore_eos; + gboolean repeat_after_eos; /* Subclasses can force an alpha channel in the (input thus output) * colorspace format */ diff --git a/tests/check/elements/compositor.c b/tests/check/elements/compositor.c index 95c518f04b..34525e98ca 100644 --- a/tests/check/elements/compositor.c +++ b/tests/check/elements/compositor.c @@ -1765,7 +1765,7 @@ _buffer_recvd (GstElement * appsink, gint * buffers_recvd) return GST_FLOW_OK; } -GST_START_TEST (test_ignore_eos) +GST_START_TEST (test_repeat_after_eos) { gboolean res; gint buffers_recvd; @@ -1793,9 +1793,9 @@ GST_START_TEST (test_ignore_eos) ck_assert_msg (res == TRUE, "Could not link compositor with appsink"); srcpad = gst_element_get_static_pad (src, "src"); sinkpad = gst_element_get_request_pad (compositor, "sink_%u"); - /* When "ignore-eos" is set, compositor will keep sending the last buffer even + /* When "repeat-after-eos" is set, compositor will keep sending the last buffer even * after EOS, so we will receive more buffers than we sent. */ - g_object_set (sinkpad, "ignore-eos", TRUE, NULL); + g_object_set (sinkpad, "repeat-after-eos", TRUE, NULL); link_res = gst_pad_link (srcpad, sinkpad); ck_assert_msg (GST_PAD_LINK_SUCCESSFUL (link_res), "videotestsrc -> " "compositor pad link failed: %i", link_res); @@ -2099,7 +2099,7 @@ compositor_suite (void) tcase_add_test (tc_chain, test_flush_start_flush_stop); tcase_add_test (tc_chain, test_segment_base_handling); tcase_add_test (tc_chain, test_obscured_skipped); - tcase_add_test (tc_chain, test_ignore_eos); + tcase_add_test (tc_chain, test_repeat_after_eos); tcase_add_test (tc_chain, test_pad_z_order); tcase_add_test (tc_chain, test_pad_numbering); tcase_add_test (tc_chain, test_start_time_zero_live_drop_0); From 8b231d502c4a04da190bcb63c9893ce904a071f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Fri, 4 May 2018 17:18:12 +0200 Subject: [PATCH 340/381] videoaggregator: Move property storage to private pad struct --- gst-libs/gst/video/gstvideoaggregator.c | 24 ++++++++++++++---------- gst-libs/gst/video/gstvideoaggregator.h | 7 ++----- 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index 29251a0048..4c267fdea7 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -65,6 +65,10 @@ enum struct _GstVideoAggregatorPadPrivate { + /* properties */ + guint zorder; + gboolean repeat_after_eos; + /* Converter, if NULL no conversion is done */ GstVideoConverter *convert; @@ -90,10 +94,10 @@ gst_video_aggregator_pad_get_property (GObject * object, guint prop_id, switch (prop_id) { case PROP_PAD_ZORDER: - g_value_set_uint (value, pad->zorder); + g_value_set_uint (value, pad->priv->zorder); break; case PROP_PAD_REPEAT_AFTER_EOS: - g_value_set_boolean (value, pad->repeat_after_eos); + g_value_set_boolean (value, pad->priv->repeat_after_eos); break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); @@ -105,7 +109,7 @@ static int pad_zorder_compare (const GstVideoAggregatorPad * pad1, const GstVideoAggregatorPad * pad2) { - return pad1->zorder - pad2->zorder; + return pad1->priv->zorder - pad2->priv->zorder; } static void @@ -119,13 +123,13 @@ gst_video_aggregator_pad_set_property (GObject * object, guint prop_id, switch (prop_id) { case PROP_PAD_ZORDER: GST_OBJECT_LOCK (vagg); - pad->zorder = g_value_get_uint (value); + pad->priv->zorder = g_value_get_uint (value); GST_ELEMENT (vagg)->sinkpads = g_list_sort (GST_ELEMENT (vagg)->sinkpads, (GCompareFunc) pad_zorder_compare); GST_OBJECT_UNLOCK (vagg); break; case PROP_PAD_REPEAT_AFTER_EOS: - pad->repeat_after_eos = g_value_get_boolean (value); + pad->priv->repeat_after_eos = g_value_get_boolean (value); break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); @@ -361,10 +365,10 @@ gst_video_aggregator_pad_init (GstVideoAggregatorPad * vaggpad) G_TYPE_INSTANCE_GET_PRIVATE (vaggpad, GST_TYPE_VIDEO_AGGREGATOR_PAD, GstVideoAggregatorPadPrivate); - vaggpad->zorder = DEFAULT_PAD_ZORDER; - vaggpad->repeat_after_eos = DEFAULT_PAD_REPEAT_AFTER_EOS; - vaggpad->aggregated_frame = NULL; + vaggpad->priv->zorder = DEFAULT_PAD_ZORDER; + vaggpad->priv->repeat_after_eos = DEFAULT_PAD_REPEAT_AFTER_EOS; vaggpad->priv->converted_buffer = NULL; + vaggpad->aggregated_frame = NULL; vaggpad->priv->convert = NULL; } @@ -1201,7 +1205,7 @@ gst_video_aggregator_fill_queues (GstVideoAggregator * vagg, continue; } } else { - if (is_eos && pad->repeat_after_eos) { + if (is_eos && pad->priv->repeat_after_eos) { eos = FALSE; GST_DEBUG_OBJECT (pad, "ignoring EOS and re-using previous buffer"); continue; @@ -1846,7 +1850,7 @@ gst_video_aggregator_request_new_pad (GstElement * element, return NULL; GST_OBJECT_LOCK (vagg); - vaggpad->zorder = GST_ELEMENT (vagg)->numsinkpads; + vaggpad->priv->zorder = GST_ELEMENT (vagg)->numsinkpads; vaggpad->priv->start_time = -1; vaggpad->priv->end_time = -1; element->sinkpads = g_list_sort (element->sinkpads, diff --git a/gst-libs/gst/video/gstvideoaggregator.h b/gst-libs/gst/video/gstvideoaggregator.h index a1b7f8d9c2..471b4c9b89 100644 --- a/gst-libs/gst/video/gstvideoaggregator.h +++ b/gst-libs/gst/video/gstvideoaggregator.h @@ -67,16 +67,13 @@ struct _GstVideoAggregatorPad { GstAggregatorPad parent; + /*< public >*/ + /* read-only, with OBJECT_LOCK */ GstVideoInfo info; GstBuffer *buffer; - GstVideoFrame *aggregated_frame; - /* properties */ - guint zorder; - gboolean repeat_after_eos; - /* Subclasses can force an alpha channel in the (input thus output) * colorspace format */ gboolean needs_alpha; From 404b802846e50d4cc631001b6ff29dfd764e0bb0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Sat, 5 May 2018 12:16:35 +0200 Subject: [PATCH 341/381] glmixer: Move frame/texture mapping/unmapping into prepare/clean_frame Previously we assumed that the texture ID is going to be valid even after unmapping the frame, as it was immediately unmapped before even being used. Now we only unmap once we're done with the texture. --- ext/gl/gstglmixer.c | 103 +++++++++++++++++++++++++------------------- 1 file changed, 59 insertions(+), 44 deletions(-) diff --git a/ext/gl/gstglmixer.c b/ext/gl/gstglmixer.c index 2ea099ee03..327da54f85 100644 --- a/ext/gl/gstglmixer.c +++ b/ext/gl/gstglmixer.c @@ -38,6 +38,10 @@ static void gst_gl_mixer_pad_get_property (GObject * object, guint prop_id, GValue * value, GParamSpec * pspec); static void gst_gl_mixer_pad_set_property (GObject * object, guint prop_id, const GValue * value, GParamSpec * pspec); +static gboolean gst_gl_mixer_pad_prepare_frame (GstVideoAggregatorPad * vpad, + GstVideoAggregator * vagg); +static void gst_gl_mixer_pad_clean_frame (GstVideoAggregatorPad * vpad, + GstVideoAggregator * vagg); enum { @@ -69,8 +73,8 @@ gst_gl_mixer_pad_class_init (GstGLMixerPadClass * klass) gobject_class->get_property = gst_gl_mixer_pad_get_property; vaggpad_class->set_info = NULL; - vaggpad_class->prepare_frame = NULL; - vaggpad_class->clean_frame = NULL; + vaggpad_class->prepare_frame = gst_gl_mixer_pad_prepare_frame; + vaggpad_class->clean_frame = gst_gl_mixer_pad_clean_frame; } static void @@ -95,6 +99,59 @@ gst_gl_mixer_pad_set_property (GObject * object, guint prop_id, } } +static gboolean +gst_gl_mixer_pad_prepare_frame (GstVideoAggregatorPad * vpad, + GstVideoAggregator * vagg) +{ + GstGLMixerPad *pad = GST_GL_MIXER_PAD (vpad); + GstGLMixer *mix = GST_GL_MIXER (vagg); + + pad->current_texture = 0; + vpad->aggregated_frame = NULL; + + if (vpad->buffer != NULL) { + GstVideoInfo gl_info; + GstVideoFrame aggregated_frame; + GstGLSyncMeta *sync_meta; + + gst_video_info_set_format (&gl_info, + GST_VIDEO_FORMAT_RGBA, + GST_VIDEO_INFO_WIDTH (&vpad->info), + GST_VIDEO_INFO_HEIGHT (&vpad->info)); + + sync_meta = gst_buffer_get_gl_sync_meta (vpad->buffer); + if (sync_meta) + gst_gl_sync_meta_wait (sync_meta, GST_GL_BASE_MIXER (mix)->context); + + if (!gst_video_frame_map (&aggregated_frame, &gl_info, vpad->buffer, + GST_MAP_READ | GST_MAP_GL)) { + GST_ERROR_OBJECT (pad, "Failed to map input frame"); + return FALSE; + } + + pad->current_texture = *(guint *) aggregated_frame.data[0]; + + vpad->aggregated_frame = g_slice_new0 (GstVideoFrame); + *vpad->aggregated_frame = aggregated_frame; + } + + return TRUE; +} + +static void +gst_gl_mixer_pad_clean_frame (GstVideoAggregatorPad * vpad, + GstVideoAggregator * vagg) +{ + GstGLMixerPad *pad = GST_GL_MIXER_PAD (vpad); + + pad->current_texture = 0; + if (vpad->aggregated_frame) { + gst_video_frame_unmap (vpad->aggregated_frame); + g_slice_free (GstVideoFrame, vpad->aggregated_frame); + vpad->aggregated_frame = NULL; + } +} + static gboolean _negotiated_caps (GstAggregator * agg, GstCaps * caps) { @@ -561,42 +618,6 @@ context_error: } } -static gboolean -gst_gl_mixer_upload_frames (GstElement * element, GstPad * sink_pad, - gpointer user_data) -{ - GstVideoAggregatorPad *vaggpad = GST_VIDEO_AGGREGATOR_PAD (sink_pad); - GstGLMixerPad *pad = GST_GL_MIXER_PAD (sink_pad); - GstGLMixer *mix = GST_GL_MIXER (element); - - pad->current_texture = 0; - if (vaggpad->buffer != NULL) { - GstVideoInfo gl_info; - GstVideoFrame gl_frame; - GstGLSyncMeta *sync_meta; - - gst_video_info_set_format (&gl_info, - GST_VIDEO_FORMAT_RGBA, - GST_VIDEO_INFO_WIDTH (&vaggpad->info), - GST_VIDEO_INFO_HEIGHT (&vaggpad->info)); - - sync_meta = gst_buffer_get_gl_sync_meta (vaggpad->buffer); - if (sync_meta) - gst_gl_sync_meta_wait (sync_meta, GST_GL_BASE_MIXER (mix)->context); - - if (!gst_video_frame_map (&gl_frame, &gl_info, vaggpad->buffer, - GST_MAP_READ | GST_MAP_GL)) { - GST_ERROR_OBJECT (pad, "Failed to map input frame"); - return FALSE; - } - - pad->current_texture = *(guint *) gl_frame.data[0]; - gst_video_frame_unmap (&gl_frame); - } - - return TRUE; -} - gboolean gst_gl_mixer_process_textures (GstGLMixer * mix, GstBuffer * outbuf) { @@ -616,12 +637,6 @@ gst_gl_mixer_process_textures (GstGLMixer * mix, GstBuffer * outbuf) out_tex = (GstGLMemory *) out_frame.map[0].memory; - if (!gst_element_foreach_sink_pad (GST_ELEMENT_CAST (mix), - gst_gl_mixer_upload_frames, NULL)) { - res = FALSE; - goto out; - } - g_mutex_lock (&priv->gl_resource_lock); if (!priv->gl_resource_ready) g_cond_wait (&priv->gl_resource_cond, &priv->gl_resource_lock); From 1c8110ab1714126123aae7cd0014c031685fc2de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Sat, 5 May 2018 15:49:17 +0200 Subject: [PATCH 342/381] videoaggregator: Move aggregated_frame and the pad buffer into the private struct The aggregated_frame is now called prepared_frame and passed to the prepare_frame and cleanup_frame virtual methods directly. For the currently queued buffer there is a method on the video aggregator pad now. --- ext/gl/gstglmixer.c | 54 ++++---- ext/gl/gstglstereomix.c | 9 +- ext/gl/gstglvideomixer.c | 5 +- gst-libs/gst/video/gstvideoaggregator.c | 159 +++++++++++++++++------- gst-libs/gst/video/gstvideoaggregator.h | 19 ++- gst/compositor/compositor.c | 98 +++++++-------- 6 files changed, 204 insertions(+), 140 deletions(-) diff --git a/ext/gl/gstglmixer.c b/ext/gl/gstglmixer.c index 327da54f85..a7d2240ce3 100644 --- a/ext/gl/gstglmixer.c +++ b/ext/gl/gstglmixer.c @@ -39,9 +39,10 @@ static void gst_gl_mixer_pad_get_property (GObject * object, guint prop_id, static void gst_gl_mixer_pad_set_property (GObject * object, guint prop_id, const GValue * value, GParamSpec * pspec); static gboolean gst_gl_mixer_pad_prepare_frame (GstVideoAggregatorPad * vpad, - GstVideoAggregator * vagg); + GstVideoAggregator * vagg, GstBuffer * buffer, + GstVideoFrame * prepared_frame); static void gst_gl_mixer_pad_clean_frame (GstVideoAggregatorPad * vpad, - GstVideoAggregator * vagg); + GstVideoAggregator * vagg, GstVideoFrame * prepared_frame); enum { @@ -101,54 +102,45 @@ gst_gl_mixer_pad_set_property (GObject * object, guint prop_id, static gboolean gst_gl_mixer_pad_prepare_frame (GstVideoAggregatorPad * vpad, - GstVideoAggregator * vagg) + GstVideoAggregator * vagg, GstBuffer * buffer, + GstVideoFrame * prepared_frame) { GstGLMixerPad *pad = GST_GL_MIXER_PAD (vpad); GstGLMixer *mix = GST_GL_MIXER (vagg); + GstVideoInfo gl_info; + GstGLSyncMeta *sync_meta; pad->current_texture = 0; - vpad->aggregated_frame = NULL; - if (vpad->buffer != NULL) { - GstVideoInfo gl_info; - GstVideoFrame aggregated_frame; - GstGLSyncMeta *sync_meta; + gst_video_info_set_format (&gl_info, + GST_VIDEO_FORMAT_RGBA, + GST_VIDEO_INFO_WIDTH (&vpad->info), GST_VIDEO_INFO_HEIGHT (&vpad->info)); - gst_video_info_set_format (&gl_info, - GST_VIDEO_FORMAT_RGBA, - GST_VIDEO_INFO_WIDTH (&vpad->info), - GST_VIDEO_INFO_HEIGHT (&vpad->info)); + sync_meta = gst_buffer_get_gl_sync_meta (buffer); + if (sync_meta) + gst_gl_sync_meta_wait (sync_meta, GST_GL_BASE_MIXER (mix)->context); - sync_meta = gst_buffer_get_gl_sync_meta (vpad->buffer); - if (sync_meta) - gst_gl_sync_meta_wait (sync_meta, GST_GL_BASE_MIXER (mix)->context); - - if (!gst_video_frame_map (&aggregated_frame, &gl_info, vpad->buffer, - GST_MAP_READ | GST_MAP_GL)) { - GST_ERROR_OBJECT (pad, "Failed to map input frame"); - return FALSE; - } - - pad->current_texture = *(guint *) aggregated_frame.data[0]; - - vpad->aggregated_frame = g_slice_new0 (GstVideoFrame); - *vpad->aggregated_frame = aggregated_frame; + if (!gst_video_frame_map (prepared_frame, &gl_info, buffer, + GST_MAP_READ | GST_MAP_GL)) { + GST_ERROR_OBJECT (pad, "Failed to map input frame"); + return FALSE; } + pad->current_texture = *(guint *) prepared_frame->data[0]; + return TRUE; } static void gst_gl_mixer_pad_clean_frame (GstVideoAggregatorPad * vpad, - GstVideoAggregator * vagg) + GstVideoAggregator * vagg, GstVideoFrame * prepared_frame) { GstGLMixerPad *pad = GST_GL_MIXER_PAD (vpad); pad->current_texture = 0; - if (vpad->aggregated_frame) { - gst_video_frame_unmap (vpad->aggregated_frame); - g_slice_free (GstVideoFrame, vpad->aggregated_frame); - vpad->aggregated_frame = NULL; + if (prepared_frame->buffer) { + gst_video_frame_unmap (prepared_frame); + memset (prepared_frame, 0, sizeof (GstVideoFrame)); } } diff --git a/ext/gl/gstglstereomix.c b/ext/gl/gstglstereomix.c index 4d72dfa3dd..e9037beaf5 100644 --- a/ext/gl/gstglstereomix.c +++ b/ext/gl/gstglstereomix.c @@ -88,7 +88,7 @@ G_DEFINE_TYPE_WITH_CODE (GstGLStereoMix, gst_gl_stereo_mix, GST_TYPE_GL_MIXER, static GstCaps *_update_caps (GstVideoAggregator * vagg, GstCaps * caps); static gboolean _negotiated_caps (GstAggregator * aggregator, GstCaps * caps); -gboolean gst_gl_stereo_mix_make_output (GstGLStereoMix * mix); +static gboolean gst_gl_stereo_mix_make_output (GstGLStereoMix * mix); static gboolean gst_gl_stereo_mix_process_frames (GstGLStereoMix * mixer); #define DEFAULT_DOWNMIX GST_GL_STEREO_DOWNMIX_ANAGLYPH_GREEN_MAGENTA_DUBOIS @@ -301,7 +301,7 @@ gst_gl_stereo_mix_get_output_buffer (GstVideoAggregator * videoaggregator, return ret; } -gboolean +static gboolean gst_gl_stereo_mix_make_output (GstGLStereoMix * mix) { GList *walk; @@ -316,11 +316,12 @@ gst_gl_stereo_mix_make_output (GstGLStereoMix * mix) while (walk) { GstVideoAggregatorPad *vaggpad = walk->data; GstGLStereoMixPad *pad = walk->data; + GstBuffer *buffer = gst_video_aggregator_pad_get_current_buffer (vaggpad); GST_LOG_OBJECT (mix, "Checking pad %" GST_PTR_FORMAT, vaggpad); - if (vaggpad->buffer != NULL) { - pad->current_buffer = vaggpad->buffer; + if (buffer != NULL) { + pad->current_buffer = buffer; GST_DEBUG_OBJECT (pad, "Got buffer %" GST_PTR_FORMAT, pad->current_buffer); diff --git a/ext/gl/gstglvideomixer.c b/ext/gl/gstglvideomixer.c index 26a42d62de..3879dbb079 100644 --- a/ext/gl/gstglvideomixer.c +++ b/ext/gl/gstglvideomixer.c @@ -1583,9 +1583,10 @@ gst_gl_video_mixer_callback (gpointer stuff) { GstVideoAffineTransformationMeta *af_meta; gfloat matrix[16]; + GstBuffer *buffer = + gst_video_aggregator_pad_get_current_buffer (vagg_pad); - af_meta = - gst_buffer_get_video_affine_transformation_meta (vagg_pad->buffer); + af_meta = gst_buffer_get_video_affine_transformation_meta (buffer); gst_gl_get_affine_transformation_meta_as_ndc_ext (af_meta, matrix); gst_gl_shader_set_uniform_matrix_4fv (video_mixer->shader, "u_transformation", 1, FALSE, matrix); diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index 4c267fdea7..cdbf260663 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -65,6 +65,9 @@ enum struct _GstVideoAggregatorPadPrivate { + GstBuffer *buffer; + GstVideoFrame prepared_frame; + /* properties */ guint zorder; gboolean repeat_after_eos; @@ -146,7 +149,7 @@ _flush_pad (GstAggregatorPad * aggpad, GstAggregator * aggregator) GstVideoAggregatorPad *pad = GST_VIDEO_AGGREGATOR_PAD (aggpad); gst_video_aggregator_reset_qos (vagg); - gst_buffer_replace (&pad->buffer, NULL); + gst_buffer_replace (&pad->priv->buffer, NULL); pad->priv->start_time = -1; pad->priv->end_time = -1; @@ -259,28 +262,26 @@ gst_video_aggregator_pad_finalize (GObject * o) static gboolean gst_video_aggregator_pad_prepare_frame (GstVideoAggregatorPad * pad, - GstVideoAggregator * vagg) + GstVideoAggregator * vagg, GstBuffer * buffer, + GstVideoFrame * prepared_frame) { - guint outsize; - GstVideoFrame *converted_frame; - GstBuffer *converted_buf = NULL; - GstVideoFrame *frame; - static GstAllocationParams params = { 0, 15, 0, 0, }; + GstVideoFrame frame; - if (!pad->buffer) + if (!pad->priv->buffer) return TRUE; - frame = g_slice_new0 (GstVideoFrame); - - if (!gst_video_frame_map (frame, &pad->info, pad->buffer, GST_MAP_READ)) { + if (!gst_video_frame_map (&frame, &pad->info, pad->priv->buffer, + GST_MAP_READ)) { GST_WARNING_OBJECT (vagg, "Could not map input buffer"); return FALSE; } if (pad->priv->convert) { + GstVideoFrame converted_frame; + GstBuffer *converted_buf = NULL; + static GstAllocationParams params = { 0, 15, 0, 0, }; gint converted_size; - - converted_frame = g_slice_new0 (GstVideoFrame); + guint outsize; /* We wait until here to set the conversion infos, in case vagg->info changed */ converted_size = pad->priv->conversion_info.size; @@ -288,37 +289,32 @@ gst_video_aggregator_pad_prepare_frame (GstVideoAggregatorPad * pad, converted_size = converted_size > outsize ? converted_size : outsize; converted_buf = gst_buffer_new_allocate (NULL, converted_size, ¶ms); - if (!gst_video_frame_map (converted_frame, &(pad->priv->conversion_info), + if (!gst_video_frame_map (&converted_frame, &(pad->priv->conversion_info), converted_buf, GST_MAP_READWRITE)) { GST_WARNING_OBJECT (vagg, "Could not map converted frame"); - g_slice_free (GstVideoFrame, converted_frame); - gst_video_frame_unmap (frame); - g_slice_free (GstVideoFrame, frame); + gst_video_frame_unmap (&frame); return FALSE; } - gst_video_converter_frame (pad->priv->convert, frame, converted_frame); + gst_video_converter_frame (pad->priv->convert, &frame, &converted_frame); pad->priv->converted_buffer = converted_buf; - gst_video_frame_unmap (frame); - g_slice_free (GstVideoFrame, frame); + gst_video_frame_unmap (&frame); + *prepared_frame = converted_frame; } else { - converted_frame = frame; + *prepared_frame = frame; } - pad->aggregated_frame = converted_frame; - return TRUE; } static void gst_video_aggregator_pad_clean_frame (GstVideoAggregatorPad * pad, - GstVideoAggregator * vagg) + GstVideoAggregator * vagg, GstVideoFrame * prepared_frame) { - if (pad->aggregated_frame) { - gst_video_frame_unmap (pad->aggregated_frame); - g_slice_free (GstVideoFrame, pad->aggregated_frame); - pad->aggregated_frame = NULL; + if (prepared_frame->buffer) { + gst_video_frame_unmap (prepared_frame); + memset (prepared_frame, 0, sizeof (GstVideoFrame)); } if (pad->priv->converted_buffer) { @@ -368,11 +364,77 @@ gst_video_aggregator_pad_init (GstVideoAggregatorPad * vaggpad) vaggpad->priv->zorder = DEFAULT_PAD_ZORDER; vaggpad->priv->repeat_after_eos = DEFAULT_PAD_REPEAT_AFTER_EOS; vaggpad->priv->converted_buffer = NULL; - vaggpad->aggregated_frame = NULL; + memset (&vaggpad->priv->prepared_frame, 0, sizeof (GstVideoFrame)); vaggpad->priv->convert = NULL; } +/** + * gst_video_aggregator_pad_has_current_buffer: + * @pad: a #GstVideoAggregatorPad + * + * Checks if the pad currently has a buffer queued that is going to be used + * for the current output frame. + * + * This must only be called from the aggregate_frames() virtual method, + * or from the prepare_frame() virtual method of the aggregator pads. + * + * Returns: %TRUE if the pad has currently a buffer queued + */ +gboolean +gst_video_aggregator_pad_has_current_buffer (GstVideoAggregatorPad * pad) +{ + g_return_val_if_fail (GST_IS_VIDEO_AGGREGATOR_PAD (pad), FALSE); + + return pad->priv->buffer != NULL; +} + +/** + * gst_video_aggregator_pad_has_current_buffer: + * @pad: a #GstVideoAggregatorPad + * + * Returns the currently queued buffer that is going to be used + * for the current output frame. + * + * This must only be called from the aggregate_frames() virtual method, + * or from the prepare_frame() virtual method of the aggregator pads. + * + * The return value is only valid until aggregate_frames() or prepare_frames() + * returns. + * + * Returns: (transfer none): The currently queued buffer + */ +GstBuffer * +gst_video_aggregator_pad_get_current_buffer (GstVideoAggregatorPad * pad) +{ + g_return_val_if_fail (GST_IS_VIDEO_AGGREGATOR_PAD (pad), NULL); + + return pad->priv->buffer; +} + +/** + * gst_video_aggregator_pad_get_prepared_frame: + * @pad: a #GstVideoAggregatorPad + * + * Returns the currently prepared video frame that has to be aggregated into + * the current output frame. + * + * This must only be called from the aggregate_frames() virtual method, + * or from the prepare_frame() virtual method of the aggregator pads. + * + * The return value is only valid until aggregate_frames() or prepare_frames() + * returns. + * + * Returns: (transfer none): The currently prepared video frame + */ +GstVideoFrame * +gst_video_aggregator_pad_get_prepared_frame (GstVideoAggregatorPad * pad) +{ + g_return_val_if_fail (GST_IS_VIDEO_AGGREGATOR_PAD (pad), NULL); + + return pad->priv->prepared_frame.buffer ? &pad->priv->prepared_frame : NULL; +} + /************************************** * GstVideoAggregator implementation * **************************************/ @@ -1017,7 +1079,7 @@ gst_video_aggregator_reset (GstVideoAggregator * vagg) for (l = GST_ELEMENT (vagg)->sinkpads; l; l = l->next) { GstVideoAggregatorPad *p = l->data; - gst_buffer_replace (&p->buffer, NULL); + gst_buffer_replace (&p->priv->buffer, NULL); p->priv->start_time = -1; p->priv->end_time = -1; @@ -1038,7 +1100,7 @@ gst_video_aggregator_fill_queues (GstVideoAggregator * vagg, gboolean need_reconfigure = FALSE; GstSegment *agg_segment = &GST_AGGREGATOR_PAD (agg->srcpad)->segment; - /* get a set of buffers into pad->buffer that are within output_start_running_time + /* get a set of buffers into pad->priv->buffer that are within output_start_running_time * and output_end_running_time taking into account finished and unresponsive pads */ GST_OBJECT_LOCK (vagg); @@ -1078,7 +1140,7 @@ gst_video_aggregator_fill_queues (GstVideoAggregator * vagg, gst_segment_to_running_time (&segment, GST_FORMAT_TIME, start_time); if (start_time >= output_end_running_time) { - if (pad->buffer) { + if (pad->priv->buffer) { GST_DEBUG_OBJECT (pad, "buffer duration is -1, start_time >= " "output_end_running_time. Keeping previous buffer"); } else { @@ -1090,7 +1152,7 @@ gst_video_aggregator_fill_queues (GstVideoAggregator * vagg, } else if (start_time < output_start_running_time) { GST_DEBUG_OBJECT (pad, "buffer duration is -1, start_time < " "output_start_running_time. Discarding old buffer"); - gst_buffer_replace (&pad->buffer, buf); + gst_buffer_replace (&pad->priv->buffer, buf); if (pad->priv->pending_vinfo.finfo) { pad->info = pad->priv->pending_vinfo; need_reconfigure = TRUE; @@ -1103,7 +1165,7 @@ gst_video_aggregator_fill_queues (GstVideoAggregator * vagg, } gst_buffer_unref (buf); buf = gst_aggregator_pad_pop_buffer (bpad); - gst_buffer_replace (&pad->buffer, buf); + gst_buffer_replace (&pad->priv->buffer, buf); if (pad->priv->pending_vinfo.finfo) { pad->info = pad->priv->pending_vinfo; need_reconfigure = TRUE; @@ -1168,7 +1230,7 @@ gst_video_aggregator_fill_queues (GstVideoAggregator * vagg, GST_DEBUG_OBJECT (pad, "Taking new buffer with start time %" GST_TIME_FORMAT, GST_TIME_ARGS (start_time)); - gst_buffer_replace (&pad->buffer, buf); + gst_buffer_replace (&pad->priv->buffer, buf); if (pad->priv->pending_vinfo.finfo) { pad->info = pad->priv->pending_vinfo; need_reconfigure = TRUE; @@ -1186,7 +1248,7 @@ gst_video_aggregator_fill_queues (GstVideoAggregator * vagg, gst_buffer_unref (buf); eos = FALSE; } else { - gst_buffer_replace (&pad->buffer, buf); + gst_buffer_replace (&pad->priv->buffer, buf); if (pad->priv->pending_vinfo.finfo) { pad->info = pad->priv->pending_vinfo; need_reconfigure = TRUE; @@ -1218,13 +1280,13 @@ gst_video_aggregator_fill_queues (GstVideoAggregator * vagg, GST_DEBUG ("I just need more data"); need_more_data = TRUE; } else { - gst_buffer_replace (&pad->buffer, NULL); + gst_buffer_replace (&pad->priv->buffer, NULL); } } else if (is_eos) { eos = FALSE; } } else if (is_eos) { - gst_buffer_replace (&pad->buffer, NULL); + gst_buffer_replace (&pad->priv->buffer, NULL); } } } @@ -1249,10 +1311,10 @@ sync_pad_values (GstElement * vagg, GstPad * pad, gpointer user_data) GstClockTime timestamp; gint64 stream_time; - if (vpad->buffer == NULL) + if (vpad->priv->buffer == NULL) return TRUE; - timestamp = GST_BUFFER_TIMESTAMP (vpad->buffer); + timestamp = GST_BUFFER_TIMESTAMP (vpad->priv->buffer); GST_OBJECT_LOCK (bpad); stream_time = gst_segment_to_stream_time (&bpad->segment, GST_FORMAT_TIME, timestamp); @@ -1272,10 +1334,13 @@ prepare_frames (GstElement * agg, GstPad * pad, gpointer user_data) GstVideoAggregatorPadClass *vaggpad_class = GST_VIDEO_AGGREGATOR_PAD_GET_CLASS (pad); - if (vpad->buffer == NULL || !vaggpad_class->prepare_frame) + memset (&vpad->priv->prepared_frame, 0, sizeof (GstVideoFrame)); + + if (vpad->priv->buffer == NULL || !vaggpad_class->prepare_frame) return TRUE; - return vaggpad_class->prepare_frame (vpad, GST_VIDEO_AGGREGATOR_CAST (agg)); + return vaggpad_class->prepare_frame (vpad, GST_VIDEO_AGGREGATOR_CAST (agg), + vpad->priv->buffer, &vpad->priv->prepared_frame); } static gboolean @@ -1287,7 +1352,9 @@ clean_pad (GstElement * agg, GstPad * pad, gpointer user_data) GST_VIDEO_AGGREGATOR_PAD_GET_CLASS (pad); if (vaggpad_class->clean_frame) - vaggpad_class->clean_frame (vpad, vagg); + vaggpad_class->clean_frame (vpad, vagg, &vpad->priv->prepared_frame); + + memset (&vpad->priv->prepared_frame, 0, sizeof (GstVideoFrame)); return TRUE; } @@ -1749,11 +1816,11 @@ gst_video_aggregator_flush (GstAggregator * agg) /* Convert to the output segment rate */ if (ABS (agg_segment->rate) != abs_rate) { - if (ABS (agg_segment->rate) != 1.0 && p->buffer) { + if (ABS (agg_segment->rate) != 1.0 && p->priv->buffer) { p->priv->start_time /= ABS (agg_segment->rate); p->priv->end_time /= ABS (agg_segment->rate); } - if (abs_rate != 1.0 && p->buffer) { + if (abs_rate != 1.0 && p->priv->buffer) { p->priv->start_time *= abs_rate; p->priv->end_time *= abs_rate; } @@ -1879,7 +1946,7 @@ gst_video_aggregator_release_pad (GstElement * element, GstPad * pad) if (last_pad) gst_video_aggregator_reset (vagg); - gst_buffer_replace (&vaggpad->buffer, NULL); + gst_buffer_replace (&vaggpad->priv->buffer, NULL); GST_ELEMENT_CLASS (gst_video_aggregator_parent_class)->release_pad (GST_ELEMENT (vagg), pad); diff --git a/gst-libs/gst/video/gstvideoaggregator.h b/gst-libs/gst/video/gstvideoaggregator.h index 471b4c9b89..af1e026828 100644 --- a/gst-libs/gst/video/gstvideoaggregator.h +++ b/gst-libs/gst/video/gstvideoaggregator.h @@ -71,9 +71,6 @@ struct _GstVideoAggregatorPad /* read-only, with OBJECT_LOCK */ GstVideoInfo info; - GstBuffer *buffer; - GstVideoFrame *aggregated_frame; - /* Subclasses can force an alpha channel in the (input thus output) * colorspace format */ gboolean needs_alpha; @@ -102,10 +99,13 @@ struct _GstVideoAggregatorPadClass GstVideoInfo * wanted_info); gboolean (*prepare_frame) (GstVideoAggregatorPad * pad, - GstVideoAggregator * videoaggregator); + GstVideoAggregator * videoaggregator, + GstBuffer * buffer, + GstVideoFrame * prepared_frame); void (*clean_frame) (GstVideoAggregatorPad * pad, - GstVideoAggregator * videoaggregator); + GstVideoAggregator * videoaggregator, + GstVideoFrame * prepared_frame); gpointer _gst_reserved[GST_PADDING_LARGE]; }; @@ -113,6 +113,15 @@ struct _GstVideoAggregatorPadClass GST_VIDEO_BAD_API GType gst_video_aggregator_pad_get_type (void); +GST_VIDEO_BAD_API +gboolean gst_video_aggregator_pad_has_current_buffer (GstVideoAggregatorPad *pad); + +GST_VIDEO_BAD_API +GstBuffer * gst_video_aggregator_pad_get_current_buffer (GstVideoAggregatorPad *pad); + +GST_VIDEO_BAD_API +GstVideoFrame * gst_video_aggregator_pad_get_prepared_frame (GstVideoAggregatorPad *pad); + #define GST_TYPE_VIDEO_AGGREGATOR (gst_video_aggregator_get_type()) #define GST_VIDEO_AGGREGATOR(obj) \ (G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_VIDEO_AGGREGATOR, GstVideoAggregator)) diff --git a/gst/compositor/compositor.c b/gst/compositor/compositor.c index 66cf36ccf5..f5c83fe772 100644 --- a/gst/compositor/compositor.c +++ b/gst/compositor/compositor.c @@ -376,14 +376,13 @@ clamp_rectangle (gint x, gint y, gint w, gint h, gint outer_width, static gboolean gst_compositor_pad_prepare_frame (GstVideoAggregatorPad * pad, - GstVideoAggregator * vagg) + GstVideoAggregator * vagg, GstBuffer * buffer, + GstVideoFrame * prepared_frame) { GstCompositor *comp = GST_COMPOSITOR (vagg); GstCompositorPad *cpad = GST_COMPOSITOR_PAD (pad); guint outsize; - GstVideoFrame *converted_frame; - GstBuffer *converted_buf = NULL; - GstVideoFrame *frame; + GstVideoFrame frame; static GstAllocationParams params = { 0, 15, 0, 0, }; gint width, height; gboolean frame_obscured = FALSE; @@ -392,9 +391,6 @@ gst_compositor_pad_prepare_frame (GstVideoAggregatorPad * pad, * Due to the clamping, this is different from the frame width/height above. */ GstVideoRectangle frame_rect; - if (!pad->buffer) - return TRUE; - /* There's three types of width/height here: * 1. GST_VIDEO_FRAME_WIDTH/HEIGHT: * The frame width/height (same as pad->info.height/width; @@ -476,7 +472,6 @@ gst_compositor_pad_prepare_frame (GstVideoAggregatorPad * pad, if (cpad->alpha == 0.0) { GST_DEBUG_OBJECT (vagg, "Pad has alpha 0.0, not converting frame"); - converted_frame = NULL; goto done; } @@ -486,7 +481,6 @@ gst_compositor_pad_prepare_frame (GstVideoAggregatorPad * pad, if (frame_rect.w == 0 || frame_rect.h == 0) { GST_DEBUG_OBJECT (vagg, "Resulting frame is zero-width or zero-height " "(w: %i, h: %i), skipping", frame_rect.w, frame_rect.h); - converted_frame = NULL; goto done; } @@ -523,9 +517,9 @@ gst_compositor_pad_prepare_frame (GstVideoAggregatorPad * pad, /* Check if there's a buffer to be aggregated, ensure it can't have an alpha * channel, then check opacity and frame boundaries */ - if (pad2->buffer && cpad2->alpha == 1.0 && - !GST_VIDEO_INFO_HAS_ALPHA (&pad2->info) && - is_rectangle_contained (frame_rect, frame2_rect)) { + if (gst_video_aggregator_pad_has_current_buffer (pad2) + && cpad2->alpha == 1.0 && !GST_VIDEO_INFO_HAS_ALPHA (&pad2->info) + && is_rectangle_contained (frame_rect, frame2_rect)) { frame_obscured = TRUE; GST_DEBUG_OBJECT (pad, "%ix%i@(%i,%i) obscured by %s %ix%i@(%i,%i) " "in output of size %ix%i; skipping frame", frame_rect.w, frame_rect.h, @@ -538,22 +532,18 @@ gst_compositor_pad_prepare_frame (GstVideoAggregatorPad * pad, } GST_OBJECT_UNLOCK (vagg); - if (frame_obscured) { - converted_frame = NULL; + if (frame_obscured) goto done; - } - frame = g_slice_new0 (GstVideoFrame); - - if (!gst_video_frame_map (frame, &pad->info, pad->buffer, GST_MAP_READ)) { + if (!gst_video_frame_map (&frame, &pad->info, buffer, GST_MAP_READ)) { GST_WARNING_OBJECT (vagg, "Could not map input buffer"); return FALSE; } if (cpad->convert) { gint converted_size; - - converted_frame = g_slice_new0 (GstVideoFrame); + GstVideoFrame converted_frame; + GstBuffer *converted_buf = NULL; /* We wait until here to set the conversion infos, in case vagg->info changed */ converted_size = GST_VIDEO_INFO_SIZE (&cpad->conversion_info); @@ -561,40 +551,36 @@ gst_compositor_pad_prepare_frame (GstVideoAggregatorPad * pad, converted_size = converted_size > outsize ? converted_size : outsize; converted_buf = gst_buffer_new_allocate (NULL, converted_size, ¶ms); - if (!gst_video_frame_map (converted_frame, &(cpad->conversion_info), + if (!gst_video_frame_map (&converted_frame, &(cpad->conversion_info), converted_buf, GST_MAP_READWRITE)) { GST_WARNING_OBJECT (vagg, "Could not map converted frame"); - g_slice_free (GstVideoFrame, converted_frame); - gst_video_frame_unmap (frame); - g_slice_free (GstVideoFrame, frame); + gst_video_frame_unmap (&frame); return FALSE; } - gst_video_converter_frame (cpad->convert, frame, converted_frame); + gst_video_converter_frame (cpad->convert, &frame, &converted_frame); cpad->converted_buffer = converted_buf; - gst_video_frame_unmap (frame); - g_slice_free (GstVideoFrame, frame); + gst_video_frame_unmap (&frame); + *prepared_frame = converted_frame; } else { - converted_frame = frame; + *prepared_frame = frame; } done: - pad->aggregated_frame = converted_frame; return TRUE; } static void gst_compositor_pad_clean_frame (GstVideoAggregatorPad * pad, - GstVideoAggregator * vagg) + GstVideoAggregator * vagg, GstVideoFrame * prepared_frame) { GstCompositorPad *cpad = GST_COMPOSITOR_PAD (pad); - if (pad->aggregated_frame) { - gst_video_frame_unmap (pad->aggregated_frame); - g_slice_free (GstVideoFrame, pad->aggregated_frame); - pad->aggregated_frame = NULL; + if (prepared_frame->buffer) { + gst_video_frame_unmap (prepared_frame); + memset (prepared_frame, 0, sizeof (GstVideoFrame)); } if (cpad->converted_buffer) { @@ -1065,44 +1051,50 @@ gst_compositor_crossfade_frames (GstCompositor * self, GstVideoFrame * outframe) for (l = GST_ELEMENT (self)->sinkpads; l; l = l->next) { GstVideoAggregatorPad *pad = l->data; GstCompositorPad *compo_pad = GST_COMPOSITOR_PAD (pad); + GstVideoFrame *prepared_frame = + gst_video_aggregator_pad_get_prepared_frame (pad); - if (compo_pad->crossfade >= 0.0f && pad->aggregated_frame) { + if (compo_pad->crossfade >= 0.0f && prepared_frame) { gfloat alpha = compo_pad->crossfade * compo_pad->alpha; GstVideoAggregatorPad *npad = l->next ? l->next->data : NULL; - GstVideoFrame *nframe; + GstVideoFrame *next_prepared_frame; + GstVideoFrame nframe; + + next_prepared_frame = + npad ? gst_video_aggregator_pad_get_prepared_frame (npad) : NULL; if (!all_crossfading) { - nframe = g_slice_new0 (GstVideoFrame); - gst_compositor_fill_transparent (self, outframe, nframe); + gst_compositor_fill_transparent (self, outframe, &nframe); } else { - nframe = outframe; + nframe = *outframe; } - self->overlay (pad->aggregated_frame, + self->overlay (prepared_frame, compo_pad->crossfaded ? 0 : compo_pad->xpos, compo_pad->crossfaded ? 0 : compo_pad->ypos, - alpha, nframe, COMPOSITOR_BLEND_MODE_ADDITIVE); + alpha, &nframe, COMPOSITOR_BLEND_MODE_ADDITIVE); - if (npad && npad->aggregated_frame) { + if (npad && next_prepared_frame) { GstCompositorPad *next_compo_pad = GST_COMPOSITOR_PAD (npad); alpha = (1.0 - compo_pad->crossfade) * next_compo_pad->alpha; - self->overlay (npad->aggregated_frame, next_compo_pad->xpos, - next_compo_pad->ypos, alpha, nframe, + self->overlay (next_prepared_frame, next_compo_pad->xpos, + next_compo_pad->ypos, alpha, &nframe, COMPOSITOR_BLEND_MODE_ADDITIVE); /* Replace frame with current frame */ - gst_compositor_pad_clean_frame (npad, vagg); - npad->aggregated_frame = !all_crossfading ? nframe : NULL; + gst_compositor_pad_clean_frame (npad, vagg, next_prepared_frame); + if (!all_crossfading) + *next_prepared_frame = nframe; next_compo_pad->crossfaded = TRUE; /* Frame is now consumed, clean it up */ - gst_compositor_pad_clean_frame (pad, vagg); - pad->aggregated_frame = NULL; + gst_compositor_pad_clean_frame (pad, vagg, prepared_frame); } else { GST_LOG_OBJECT (self, "Simply fading out as no following pad found"); - gst_compositor_pad_clean_frame (pad, vagg); - pad->aggregated_frame = !all_crossfading ? nframe : NULL; + gst_compositor_pad_clean_frame (pad, vagg, prepared_frame); + if (!all_crossfading) + *prepared_frame = nframe; compo_pad->crossfaded = TRUE; } } @@ -1156,9 +1148,11 @@ gst_compositor_aggregate_frames (GstVideoAggregator * vagg, GstBuffer * outbuf) for (l = GST_ELEMENT (vagg)->sinkpads; l; l = l->next) { GstVideoAggregatorPad *pad = l->data; GstCompositorPad *compo_pad = GST_COMPOSITOR_PAD (pad); + GstVideoFrame *prepared_frame = + gst_video_aggregator_pad_get_prepared_frame (pad); - if (pad->aggregated_frame != NULL) { - composite (pad->aggregated_frame, + if (prepared_frame != NULL) { + composite (prepared_frame, compo_pad->crossfaded ? 0 : compo_pad->xpos, compo_pad->crossfaded ? 0 : compo_pad->ypos, compo_pad->alpha, outframe, COMPOSITOR_BLEND_MODE_NORMAL); From 233ee9f0773f051269ccbd1c283c8d4628891992 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Sat, 5 May 2018 16:14:14 +0200 Subject: [PATCH 343/381] videoaggregator: Move needs_alpha pad field to the private struct And also trigger renegotiation if the value has changed. https://bugzilla.gnome.org/show_bug.cgi?id=795836 --- gst-libs/gst/video/gstvideoaggregator.c | 32 ++++++++++++++++++++++++- gst-libs/gst/video/gstvideoaggregator.h | 7 +++--- gst/compositor/compositor.c | 3 ++- 3 files changed, 36 insertions(+), 6 deletions(-) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index cdbf260663..580e73713e 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -72,6 +72,10 @@ struct _GstVideoAggregatorPadPrivate guint zorder; gboolean repeat_after_eos; + /* Subclasses can force an alpha channel in the (input thus output) + * colorspace format */ + gboolean needs_alpha; + /* Converter, if NULL no conversion is done */ GstVideoConverter *convert; @@ -435,6 +439,32 @@ gst_video_aggregator_pad_get_prepared_frame (GstVideoAggregatorPad * pad) return pad->priv->prepared_frame.buffer ? &pad->priv->prepared_frame : NULL; } +/** + * gst_video_aggregator_pad_set_needs_alpha: + * @pad: a #GstVideoAggregatorPad + * @needs_alpha: %TRUE if this pad requires alpha output + * + * Allows selecting that this pad requires an output format with alpha + * + * Returns: (transfer none): The currently prepared video frame + */ +void +gst_video_aggregator_pad_set_needs_alpha (GstVideoAggregatorPad * pad, + gboolean needs_alpha) +{ + g_return_if_fail (GST_IS_VIDEO_AGGREGATOR_PAD (pad)); + + if (needs_alpha != pad->priv->needs_alpha) { + GstAggregator *agg = + GST_AGGREGATOR (gst_object_get_parent (GST_OBJECT (pad))); + pad->priv->needs_alpha = needs_alpha; + if (agg) { + gst_pad_mark_reconfigure (GST_AGGREGATOR_SRC_PAD (agg)); + gst_object_unref (agg); + } + } +} + /************************************** * GstVideoAggregator implementation * **************************************/ @@ -562,7 +592,7 @@ gst_video_aggregator_find_best_format (GstVideoAggregator * vagg, GINT_TO_POINTER (format_number)); /* If that pad is the first with alpha, set it as the new best format */ - if (!need_alpha && (pad->needs_alpha + if (!need_alpha && (pad->priv->needs_alpha && (!GST_VIDEO_FORMAT_INFO_HAS_ALPHA (pad->info.finfo)))) { need_alpha = TRUE; /* Just fallback to ARGB in case we require alpha but the input pad diff --git a/gst-libs/gst/video/gstvideoaggregator.h b/gst-libs/gst/video/gstvideoaggregator.h index af1e026828..70715ca1fa 100644 --- a/gst-libs/gst/video/gstvideoaggregator.h +++ b/gst-libs/gst/video/gstvideoaggregator.h @@ -71,10 +71,6 @@ struct _GstVideoAggregatorPad /* read-only, with OBJECT_LOCK */ GstVideoInfo info; - /* Subclasses can force an alpha channel in the (input thus output) - * colorspace format */ - gboolean needs_alpha; - /* < private > */ GstVideoAggregatorPadPrivate *priv; @@ -122,6 +118,9 @@ GstBuffer * gst_video_aggregator_pad_get_current_buffer (GstVideoAggregatorPad * GST_VIDEO_BAD_API GstVideoFrame * gst_video_aggregator_pad_get_prepared_frame (GstVideoAggregatorPad *pad); +GST_VIDEO_BAD_API +void gst_video_aggregator_pad_set_needs_alpha (GstVideoAggregatorPad *pad, gboolean needs_alpha); + #define GST_TYPE_VIDEO_AGGREGATOR (gst_video_aggregator_get_type()) #define GST_VIDEO_AGGREGATOR(obj) \ (G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_VIDEO_AGGREGATOR, GstVideoAggregator)) diff --git a/gst/compositor/compositor.c b/gst/compositor/compositor.c index f5c83fe772..2c5b76bdb6 100644 --- a/gst/compositor/compositor.c +++ b/gst/compositor/compositor.c @@ -198,7 +198,8 @@ gst_compositor_pad_set_property (GObject * object, guint prop_id, break; case PROP_PAD_CROSSFADE_RATIO: pad->crossfade = g_value_get_double (value); - GST_VIDEO_AGGREGATOR_PAD (pad)->needs_alpha = pad->crossfade >= 0.0f; + gst_video_aggregator_pad_set_needs_alpha (GST_VIDEO_AGGREGATOR_PAD (pad), + pad->crossfade >= 0.0f); break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); From b73b7230d2a908fe03308a60fc3fd758e2658513 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Sat, 5 May 2018 16:31:13 +0200 Subject: [PATCH 344/381] videoaggregator: Remove sink_non_alpha_caps class field This is only used for caching reasons and should never actually be in the public API. If this is ever a bottleneck later, caching around a class private struct could be implemented. --- gst-libs/gst/video/gstvideoaggregator.c | 156 +++++++++++------------- gst-libs/gst/video/gstvideoaggregator.h | 2 - 2 files changed, 68 insertions(+), 90 deletions(-) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index 580e73713e..087e0988aa 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -972,6 +972,69 @@ gst_video_aggregator_caps_has_alpha (GstCaps * caps) return FALSE; } +static GstCaps * +_get_non_alpha_caps (GstCaps * caps) +{ + GstCaps *result; + guint i, size; + + size = gst_caps_get_size (caps); + result = gst_caps_new_empty (); + for (i = 0; i < size; i++) { + GstStructure *s = gst_caps_get_structure (caps, i); + const GValue *formats = gst_structure_get_value (s, "format"); + GValue new_formats = { 0, }; + gboolean has_format = FALSE; + + /* FIXME what to do if formats are missing? */ + if (formats) { + const GstVideoFormatInfo *info; + + if (GST_VALUE_HOLDS_LIST (formats)) { + guint list_size = gst_value_list_get_size (formats); + guint index; + + g_value_init (&new_formats, GST_TYPE_LIST); + + for (index = 0; index < list_size; index++) { + const GValue *list_item = gst_value_list_get_value (formats, index); + + info = + gst_video_format_get_info (gst_video_format_from_string + (g_value_get_string (list_item))); + if (!GST_VIDEO_FORMAT_INFO_HAS_ALPHA (info)) { + has_format = TRUE; + gst_value_list_append_value (&new_formats, list_item); + } + } + + } else if (G_VALUE_HOLDS_STRING (formats)) { + info = + gst_video_format_get_info (gst_video_format_from_string + (g_value_get_string (formats))); + if (!GST_VIDEO_FORMAT_INFO_HAS_ALPHA (info)) { + has_format = TRUE; + gst_value_init_and_copy (&new_formats, formats); + } + + } else { + g_assert_not_reached (); + GST_WARNING ("Unexpected type for video 'format' field: %s", + G_VALUE_TYPE_NAME (formats)); + } + + if (has_format) { + s = gst_structure_copy (s); + gst_structure_take_value (s, "format", &new_formats); + gst_caps_append_structure (result, s); + } + + } + } + + return result; +} + static GstCaps * gst_video_aggregator_pad_sink_getcaps (GstPad * pad, GstVideoAggregator * vagg, GstCaps * filter) @@ -1020,11 +1083,11 @@ gst_video_aggregator_pad_sink_getcaps (GstPad * pad, GstVideoAggregator * vagg, returned_caps = srccaps; } - if (has_alpha) { - sink_template_caps = gst_pad_get_pad_template_caps (pad); - } else { - GstVideoAggregatorClass *klass = GST_VIDEO_AGGREGATOR_GET_CLASS (vagg); - sink_template_caps = gst_caps_ref (klass->sink_non_alpha_caps); + sink_template_caps = gst_pad_get_pad_template_caps (pad); + if (!has_alpha) { + GstCaps *tmp = _get_non_alpha_caps (sink_template_caps); + gst_caps_unref (sink_template_caps); + sink_template_caps = tmp; } { @@ -2288,83 +2351,10 @@ gst_video_aggregator_class_init (GstVideoAggregatorClass * klass) g_type_class_ref (GST_TYPE_VIDEO_AGGREGATOR_PAD); } -static inline GstCaps * -_get_non_alpha_caps_from_template (GstVideoAggregatorClass * klass) -{ - GstCaps *result; - GstCaps *templatecaps; - guint i, size; - - templatecaps = - gst_pad_template_get_caps (gst_element_class_get_pad_template - (GST_ELEMENT_CLASS (klass), "sink_%u")); - - size = gst_caps_get_size (templatecaps); - result = gst_caps_new_empty (); - for (i = 0; i < size; i++) { - GstStructure *s = gst_caps_get_structure (templatecaps, i); - const GValue *formats = gst_structure_get_value (s, "format"); - GValue new_formats = { 0, }; - gboolean has_format = FALSE; - - /* FIXME what to do if formats are missing? */ - if (formats) { - const GstVideoFormatInfo *info; - - if (GST_VALUE_HOLDS_LIST (formats)) { - guint list_size = gst_value_list_get_size (formats); - guint index; - - g_value_init (&new_formats, GST_TYPE_LIST); - - for (index = 0; index < list_size; index++) { - const GValue *list_item = gst_value_list_get_value (formats, index); - - info = - gst_video_format_get_info (gst_video_format_from_string - (g_value_get_string (list_item))); - if (!GST_VIDEO_FORMAT_INFO_HAS_ALPHA (info)) { - has_format = TRUE; - gst_value_list_append_value (&new_formats, list_item); - } - } - - } else if (G_VALUE_HOLDS_STRING (formats)) { - info = - gst_video_format_get_info (gst_video_format_from_string - (g_value_get_string (formats))); - if (!GST_VIDEO_FORMAT_INFO_HAS_ALPHA (info)) { - has_format = TRUE; - gst_value_init_and_copy (&new_formats, formats); - } - - } else { - g_assert_not_reached (); - GST_WARNING ("Unexpected type for video 'format' field: %s", - G_VALUE_TYPE_NAME (formats)); - } - - if (has_format) { - s = gst_structure_copy (s); - gst_structure_take_value (s, "format", &new_formats); - gst_caps_append_structure (result, s); - } - - } - } - - gst_caps_unref (templatecaps); - - return result; -} - -static GMutex sink_caps_mutex; - static void gst_video_aggregator_init (GstVideoAggregator * vagg, GstVideoAggregatorClass * klass) { - vagg->priv = G_TYPE_INSTANCE_GET_PRIVATE (vagg, GST_TYPE_VIDEO_AGGREGATOR, GstVideoAggregatorPrivate); @@ -2374,15 +2364,5 @@ gst_video_aggregator_init (GstVideoAggregator * vagg, g_mutex_init (&vagg->priv->lock); /* initialize variables */ - g_mutex_lock (&sink_caps_mutex); - if (klass->sink_non_alpha_caps == NULL) { - klass->sink_non_alpha_caps = _get_non_alpha_caps_from_template (klass); - - /* The caps is cached */ - GST_MINI_OBJECT_FLAG_SET (klass->sink_non_alpha_caps, - GST_MINI_OBJECT_FLAG_MAY_BE_LEAKED); - } - g_mutex_unlock (&sink_caps_mutex); - gst_video_aggregator_reset (vagg); } diff --git a/gst-libs/gst/video/gstvideoaggregator.h b/gst-libs/gst/video/gstvideoaggregator.h index 70715ca1fa..8b815f4ab8 100644 --- a/gst-libs/gst/video/gstvideoaggregator.h +++ b/gst-libs/gst/video/gstvideoaggregator.h @@ -188,8 +188,6 @@ struct _GstVideoAggregatorClass GstVideoInfo * best_info, gboolean * at_least_one_alpha); - GstCaps *sink_non_alpha_caps; - /* < private > */ gpointer _gst_reserved[GST_PADDING_LARGE]; }; From d37e778692c2e73155c38a2e8e5390490ffaa3ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Sat, 5 May 2018 17:47:55 +0200 Subject: [PATCH 345/381] glmixer: Include string.h for memset() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit gstglmixer.c:143:5: error: implicit declaration of function ‘memset’ [-Werror=implicit-function-declaration] memset (prepared_frame, 0, sizeof (GstVideoFrame)); ^~~~~~ gstglmixer.c:143:5: error: incompatible implicit declaration of built-in function ‘memset’ [-Werror] --- ext/gl/gstglmixer.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ext/gl/gstglmixer.c b/ext/gl/gstglmixer.c index a7d2240ce3..b4602eab75 100644 --- a/ext/gl/gstglmixer.c +++ b/ext/gl/gstglmixer.c @@ -26,6 +26,8 @@ #include #include +#include + #include "gstglmixer.h" #define gst_gl_mixer_parent_class parent_class From 33a8e28207f70f36060a31d183fdc512cc10ce52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Sun, 6 May 2018 15:21:24 +0200 Subject: [PATCH 346/381] videoaggregator: Switch to a GstVideoAggregatorConvertPad subclass This moves all the conversion related code to a single place, allows less code-duplication inside compositor and makes the glmixer code less awkward. It's also the same pattern as used by GstAudioAggregator. --- ext/gl/gstglbasemixer.c | 1 - ext/gl/gstglmixer.c | 1 - gst-libs/gst/video/gstvideoaggregator.c | 394 ++++++++++++++++-------- gst-libs/gst/video/gstvideoaggregator.h | 60 +++- gst/compositor/compositor.c | 272 ++++------------ gst/compositor/compositorpad.h | 8 +- 6 files changed, 369 insertions(+), 367 deletions(-) diff --git a/ext/gl/gstglbasemixer.c b/ext/gl/gstglbasemixer.c index 87cea2c4b2..0dd3f81695 100644 --- a/ext/gl/gstglbasemixer.c +++ b/ext/gl/gstglbasemixer.c @@ -73,7 +73,6 @@ gst_gl_base_mixer_pad_class_init (GstGLBaseMixerPadClass * klass) gobject_class->set_property = gst_gl_base_mixer_pad_set_property; gobject_class->get_property = gst_gl_base_mixer_pad_get_property; - vaggpad_class->set_info = NULL; vaggpad_class->prepare_frame = NULL; vaggpad_class->clean_frame = NULL; } diff --git a/ext/gl/gstglmixer.c b/ext/gl/gstglmixer.c index b4602eab75..fcc95427c7 100644 --- a/ext/gl/gstglmixer.c +++ b/ext/gl/gstglmixer.c @@ -75,7 +75,6 @@ gst_gl_mixer_pad_class_init (GstGLMixerPadClass * klass) gobject_class->set_property = gst_gl_mixer_pad_set_property; gobject_class->get_property = gst_gl_mixer_pad_get_property; - vaggpad_class->set_info = NULL; vaggpad_class->prepare_frame = gst_gl_mixer_pad_prepare_frame; vaggpad_class->clean_frame = gst_gl_mixer_pad_clean_frame; } diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index 087e0988aa..eb86b4c70c 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -76,13 +76,6 @@ struct _GstVideoAggregatorPadPrivate * colorspace format */ gboolean needs_alpha; - /* Converter, if NULL no conversion is done */ - GstVideoConverter *convert; - - /* caps used for conversion if needed */ - GstVideoInfo conversion_info; - GstBuffer *converted_buffer; - GstClockTime start_time; GstClockTime end_time; @@ -160,75 +153,6 @@ _flush_pad (GstAggregatorPad * aggpad, GstAggregator * aggregator) return GST_FLOW_OK; } -static gboolean -gst_video_aggregator_pad_set_info (GstVideoAggregatorPad * pad, - GstVideoAggregator * vagg G_GNUC_UNUSED, - GstVideoInfo * current_info, GstVideoInfo * wanted_info) -{ - gchar *colorimetry, *best_colorimetry; - const gchar *chroma, *best_chroma; - - if (!current_info->finfo) - return TRUE; - - if (GST_VIDEO_INFO_FORMAT (current_info) == GST_VIDEO_FORMAT_UNKNOWN) - return TRUE; - - if (pad->priv->convert) - gst_video_converter_free (pad->priv->convert); - - pad->priv->convert = NULL; - - colorimetry = gst_video_colorimetry_to_string (&(current_info->colorimetry)); - chroma = gst_video_chroma_to_string (current_info->chroma_site); - - best_colorimetry = - gst_video_colorimetry_to_string (&(wanted_info->colorimetry)); - best_chroma = gst_video_chroma_to_string (wanted_info->chroma_site); - - if (GST_VIDEO_INFO_FORMAT (wanted_info) != - GST_VIDEO_INFO_FORMAT (current_info) - || g_strcmp0 (colorimetry, best_colorimetry) - || g_strcmp0 (chroma, best_chroma)) { - GstVideoInfo tmp_info; - - /* Initialize with the wanted video format and our original width and - * height as we don't want to rescale. Then copy over the wanted - * colorimetry, and chroma-site and our current pixel-aspect-ratio - * and other relevant fields. - */ - gst_video_info_set_format (&tmp_info, GST_VIDEO_INFO_FORMAT (wanted_info), - current_info->width, current_info->height); - tmp_info.chroma_site = wanted_info->chroma_site; - tmp_info.colorimetry = wanted_info->colorimetry; - tmp_info.par_n = current_info->par_n; - tmp_info.par_d = current_info->par_d; - tmp_info.fps_n = current_info->fps_n; - tmp_info.fps_d = current_info->fps_d; - tmp_info.flags = current_info->flags; - tmp_info.interlace_mode = current_info->interlace_mode; - - GST_DEBUG_OBJECT (pad, "This pad will be converted from %d to %d", - GST_VIDEO_INFO_FORMAT (current_info), - GST_VIDEO_INFO_FORMAT (&tmp_info)); - pad->priv->convert = - gst_video_converter_new (current_info, &tmp_info, NULL); - pad->priv->conversion_info = tmp_info; - if (!pad->priv->convert) { - g_free (colorimetry); - g_free (best_colorimetry); - GST_WARNING_OBJECT (pad, "No path found for conversion"); - return FALSE; - } - } else { - GST_DEBUG_OBJECT (pad, "This pad will not need conversion"); - } - g_free (colorimetry); - g_free (best_colorimetry); - - return TRUE; -} - static gboolean gst_video_aggregator_pad_skip_buffer (GstAggregatorPad * aggpad, GstAggregator * agg, GstBuffer * buffer) @@ -252,63 +176,16 @@ gst_video_aggregator_pad_skip_buffer (GstAggregatorPad * aggpad, return ret; } -static void -gst_video_aggregator_pad_finalize (GObject * o) -{ - GstVideoAggregatorPad *vaggpad = GST_VIDEO_AGGREGATOR_PAD (o); - - if (vaggpad->priv->convert) - gst_video_converter_free (vaggpad->priv->convert); - vaggpad->priv->convert = NULL; - - G_OBJECT_CLASS (gst_video_aggregator_pad_parent_class)->finalize (o); -} - static gboolean gst_video_aggregator_pad_prepare_frame (GstVideoAggregatorPad * pad, GstVideoAggregator * vagg, GstBuffer * buffer, GstVideoFrame * prepared_frame) { - GstVideoFrame frame; - - if (!pad->priv->buffer) - return TRUE; - - if (!gst_video_frame_map (&frame, &pad->info, pad->priv->buffer, - GST_MAP_READ)) { + if (!gst_video_frame_map (prepared_frame, &pad->info, buffer, GST_MAP_READ)) { GST_WARNING_OBJECT (vagg, "Could not map input buffer"); return FALSE; } - if (pad->priv->convert) { - GstVideoFrame converted_frame; - GstBuffer *converted_buf = NULL; - static GstAllocationParams params = { 0, 15, 0, 0, }; - gint converted_size; - guint outsize; - - /* We wait until here to set the conversion infos, in case vagg->info changed */ - converted_size = pad->priv->conversion_info.size; - outsize = GST_VIDEO_INFO_SIZE (&vagg->info); - converted_size = converted_size > outsize ? converted_size : outsize; - converted_buf = gst_buffer_new_allocate (NULL, converted_size, ¶ms); - - if (!gst_video_frame_map (&converted_frame, &(pad->priv->conversion_info), - converted_buf, GST_MAP_READWRITE)) { - GST_WARNING_OBJECT (vagg, "Could not map converted frame"); - - gst_video_frame_unmap (&frame); - return FALSE; - } - - gst_video_converter_frame (pad->priv->convert, &frame, &converted_frame); - pad->priv->converted_buffer = converted_buf; - gst_video_frame_unmap (&frame); - *prepared_frame = converted_frame; - } else { - *prepared_frame = frame; - } - return TRUE; } @@ -320,11 +197,6 @@ gst_video_aggregator_pad_clean_frame (GstVideoAggregatorPad * pad, gst_video_frame_unmap (prepared_frame); memset (prepared_frame, 0, sizeof (GstVideoFrame)); } - - if (pad->priv->converted_buffer) { - gst_buffer_unref (pad->priv->converted_buffer); - pad->priv->converted_buffer = NULL; - } } static void @@ -335,7 +207,6 @@ gst_video_aggregator_pad_class_init (GstVideoAggregatorPadClass * klass) gobject_class->set_property = gst_video_aggregator_pad_set_property; gobject_class->get_property = gst_video_aggregator_pad_get_property; - gobject_class->finalize = gst_video_aggregator_pad_finalize; g_object_class_install_property (gobject_class, PROP_PAD_ZORDER, g_param_spec_uint ("zorder", "Z-Order", "Z Order of the picture", @@ -352,7 +223,6 @@ gst_video_aggregator_pad_class_init (GstVideoAggregatorPadClass * klass) aggpadclass->flush = GST_DEBUG_FUNCPTR (_flush_pad); aggpadclass->skip_buffer = GST_DEBUG_FUNCPTR (gst_video_aggregator_pad_skip_buffer); - klass->set_info = GST_DEBUG_FUNCPTR (gst_video_aggregator_pad_set_info); klass->prepare_frame = GST_DEBUG_FUNCPTR (gst_video_aggregator_pad_prepare_frame); klass->clean_frame = GST_DEBUG_FUNCPTR (gst_video_aggregator_pad_clean_frame); @@ -367,10 +237,7 @@ gst_video_aggregator_pad_init (GstVideoAggregatorPad * vaggpad) vaggpad->priv->zorder = DEFAULT_PAD_ZORDER; vaggpad->priv->repeat_after_eos = DEFAULT_PAD_REPEAT_AFTER_EOS; - vaggpad->priv->converted_buffer = NULL; memset (&vaggpad->priv->prepared_frame, 0, sizeof (GstVideoFrame)); - - vaggpad->priv->convert = NULL; } /** @@ -465,6 +332,260 @@ gst_video_aggregator_pad_set_needs_alpha (GstVideoAggregatorPad * pad, } } +/**************************************** + * GstVideoAggregatorConvertPad implementation * + ****************************************/ + +struct _GstVideoAggregatorConvertPadPrivate +{ + /* Converter, if NULL no conversion is done */ + GstVideoConverter *convert; + + /* caps used for conversion if needed */ + GstVideoInfo conversion_info; + GstBuffer *converted_buffer; + + gboolean converter_config_changed; +}; + +G_DEFINE_TYPE (GstVideoAggregatorConvertPad, gst_video_aggregator_convert_pad, + GST_TYPE_VIDEO_AGGREGATOR_PAD); + +static void +gst_video_aggregator_convert_pad_finalize (GObject * o) +{ + GstVideoAggregatorConvertPad *vaggpad = GST_VIDEO_AGGREGATOR_CONVERT_PAD (o); + + if (vaggpad->priv->convert) + gst_video_converter_free (vaggpad->priv->convert); + vaggpad->priv->convert = NULL; + + G_OBJECT_CLASS (gst_video_aggregator_pad_parent_class)->finalize (o); +} + +static void + gst_video_aggregator_convert_pad_update_conversion_info_internal + (GstVideoAggregatorPad * vpad) +{ + GstVideoAggregatorConvertPad *pad = GST_VIDEO_AGGREGATOR_CONVERT_PAD (vpad); + + pad->priv->converter_config_changed = TRUE; +} + +static gboolean +gst_video_aggregator_convert_pad_prepare_frame (GstVideoAggregatorPad * vpad, + GstVideoAggregator * vagg, GstBuffer * buffer, + GstVideoFrame * prepared_frame) +{ + GstVideoAggregatorConvertPad *pad = GST_VIDEO_AGGREGATOR_CONVERT_PAD (vpad); + GstVideoFrame frame; + + /* Update/create converter as needed */ + if (pad->priv->converter_config_changed) { + GstVideoAggregatorConvertPadClass *klass = + GST_VIDEO_AGGREGATOR_CONVERT_PAD_GET_CLASS (pad); + GstVideoInfo conversion_info; + + gst_video_info_init (&conversion_info); + klass->create_conversion_info (pad, vagg, &conversion_info); + if (conversion_info.finfo == NULL) + return FALSE; + pad->priv->converter_config_changed = FALSE; + + if (!pad->priv->conversion_info.finfo + || !gst_video_info_is_equal (&conversion_info, + &pad->priv->conversion_info)) { + pad->priv->conversion_info = conversion_info; + + if (pad->priv->convert) + gst_video_converter_free (pad->priv->convert); + pad->priv->convert = NULL; + + if (!gst_video_info_is_equal (&vpad->info, &pad->priv->conversion_info)) { + pad->priv->convert = + gst_video_converter_new (&vpad->info, &pad->priv->conversion_info, + NULL); + if (!pad->priv->convert) { + GST_WARNING_OBJECT (pad, "No path found for conversion"); + return FALSE; + } + + GST_DEBUG_OBJECT (pad, "This pad will be converted from %d to %d", + GST_VIDEO_INFO_FORMAT (&vpad->info), + GST_VIDEO_INFO_FORMAT (&pad->priv->conversion_info)); + } else { + GST_DEBUG_OBJECT (pad, "This pad will not need conversion"); + } + } + } + + if (!gst_video_frame_map (&frame, &vpad->info, buffer, GST_MAP_READ)) { + GST_WARNING_OBJECT (vagg, "Could not map input buffer"); + return FALSE; + } + + if (pad->priv->convert) { + GstVideoFrame converted_frame; + GstBuffer *converted_buf = NULL; + static GstAllocationParams params = { 0, 15, 0, 0, }; + gint converted_size; + guint outsize; + + /* We wait until here to set the conversion infos, in case vagg->info changed */ + converted_size = pad->priv->conversion_info.size; + outsize = GST_VIDEO_INFO_SIZE (&vagg->info); + converted_size = converted_size > outsize ? converted_size : outsize; + converted_buf = gst_buffer_new_allocate (NULL, converted_size, ¶ms); + + if (!gst_video_frame_map (&converted_frame, &(pad->priv->conversion_info), + converted_buf, GST_MAP_READWRITE)) { + GST_WARNING_OBJECT (vagg, "Could not map converted frame"); + + gst_video_frame_unmap (&frame); + return FALSE; + } + + gst_video_converter_frame (pad->priv->convert, &frame, &converted_frame); + pad->priv->converted_buffer = converted_buf; + gst_video_frame_unmap (&frame); + *prepared_frame = converted_frame; + } else { + *prepared_frame = frame; + } + + return TRUE; +} + +static void +gst_video_aggregator_convert_pad_clean_frame (GstVideoAggregatorPad * vpad, + GstVideoAggregator * vagg, GstVideoFrame * prepared_frame) +{ + GstVideoAggregatorConvertPad *pad = GST_VIDEO_AGGREGATOR_CONVERT_PAD (vpad); + + if (prepared_frame->buffer) { + gst_video_frame_unmap (prepared_frame); + memset (prepared_frame, 0, sizeof (GstVideoFrame)); + } + + if (pad->priv->converted_buffer) { + gst_buffer_unref (pad->priv->converted_buffer); + pad->priv->converted_buffer = NULL; + } +} + +static void + gst_video_aggregator_convert_pad_create_conversion_info + (GstVideoAggregatorConvertPad * pad, GstVideoAggregator * agg, + GstVideoInfo * convert_info) +{ + GstVideoAggregatorPad *vpad = GST_VIDEO_AGGREGATOR_PAD (pad); + gchar *colorimetry, *best_colorimetry; + const gchar *chroma, *best_chroma; + + g_return_if_fail (GST_IS_VIDEO_AGGREGATOR_CONVERT_PAD (pad)); + g_return_if_fail (convert_info != NULL); + + if (!vpad->info.finfo + || GST_VIDEO_INFO_FORMAT (&vpad->info) == GST_VIDEO_FORMAT_UNKNOWN) { + return; + } + + if (!agg->info.finfo + || GST_VIDEO_INFO_FORMAT (&agg->info) == GST_VIDEO_FORMAT_UNKNOWN) { + return; + } + + colorimetry = gst_video_colorimetry_to_string (&vpad->info.colorimetry); + chroma = gst_video_chroma_to_string (vpad->info.chroma_site); + + best_colorimetry = gst_video_colorimetry_to_string (&agg->info.colorimetry); + best_chroma = gst_video_chroma_to_string (agg->info.chroma_site); + + if (GST_VIDEO_INFO_FORMAT (&agg->info) != GST_VIDEO_INFO_FORMAT (&vpad->info) + || g_strcmp0 (colorimetry, best_colorimetry) + || g_strcmp0 (chroma, best_chroma)) { + GstVideoInfo tmp_info; + + /* Initialize with the wanted video format and our original width and + * height as we don't want to rescale. Then copy over the wanted + * colorimetry, and chroma-site and our current pixel-aspect-ratio + * and other relevant fields. + */ + gst_video_info_set_format (&tmp_info, GST_VIDEO_INFO_FORMAT (&agg->info), + vpad->info.width, vpad->info.height); + tmp_info.chroma_site = agg->info.chroma_site; + tmp_info.colorimetry = agg->info.colorimetry; + tmp_info.par_n = vpad->info.par_n; + tmp_info.par_d = vpad->info.par_d; + tmp_info.fps_n = vpad->info.fps_n; + tmp_info.fps_d = vpad->info.fps_d; + tmp_info.flags = vpad->info.flags; + tmp_info.interlace_mode = vpad->info.interlace_mode; + + *convert_info = tmp_info; + } else { + *convert_info = vpad->info; + } + + g_free (colorimetry); + g_free (best_colorimetry); +} + +static void +gst_video_aggregator_convert_pad_class_init (GstVideoAggregatorConvertPadClass * + klass) +{ + GObjectClass *gobject_class = (GObjectClass *) klass; + GstVideoAggregatorPadClass *vaggpadclass = + (GstVideoAggregatorPadClass *) klass; + + gobject_class->finalize = gst_video_aggregator_convert_pad_finalize; + + g_type_class_add_private (klass, + sizeof (GstVideoAggregatorConvertPadPrivate)); + + vaggpadclass->update_conversion_info = + GST_DEBUG_FUNCPTR + (gst_video_aggregator_convert_pad_update_conversion_info_internal); + vaggpadclass->prepare_frame = + GST_DEBUG_FUNCPTR (gst_video_aggregator_convert_pad_prepare_frame); + vaggpadclass->clean_frame = + GST_DEBUG_FUNCPTR (gst_video_aggregator_convert_pad_clean_frame); + + klass->create_conversion_info = + gst_video_aggregator_convert_pad_create_conversion_info; +} + +static void +gst_video_aggregator_convert_pad_init (GstVideoAggregatorConvertPad * vaggpad) +{ + vaggpad->priv = + G_TYPE_INSTANCE_GET_PRIVATE (vaggpad, + GST_TYPE_VIDEO_AGGREGATOR_CONVERT_PAD, + GstVideoAggregatorConvertPadPrivate); + + vaggpad->priv->converted_buffer = NULL; + vaggpad->priv->convert = NULL; + vaggpad->priv->converter_config_changed = FALSE; +} + + +/** + * gst_video_aggregator_convert_pad_update_conversion_info: + * @pad: a #GstVideoAggregatorPad + * + * Requests the pad to check and update the converter before the next usage to + * update for any changes that have happened. + * + */ +void gst_video_aggregator_convert_pad_update_conversion_info + (GstVideoAggregatorConvertPad * pad) +{ + g_return_if_fail (GST_IS_VIDEO_AGGREGATOR_CONVERT_PAD (pad)); + + pad->priv->converter_config_changed = TRUE; +} + /************************************** * GstVideoAggregator implementation * **************************************/ @@ -818,9 +939,8 @@ gst_video_aggregator_default_negotiated_src_caps (GstAggregator * agg, GstVideoAggregatorPadClass *vaggpad_klass = GST_VIDEO_AGGREGATOR_PAD_GET_CLASS (pad); - if (vaggpad_klass->set_info - && !vaggpad_klass->set_info (pad, vagg, &pad->info, &vagg->info)) { - return FALSE; + if (vaggpad_klass->update_conversion_info) { + vaggpad_klass->update_conversion_info (pad); } } diff --git a/gst-libs/gst/video/gstvideoaggregator.h b/gst-libs/gst/video/gstvideoaggregator.h index 8b815f4ab8..611bd9c56a 100644 --- a/gst-libs/gst/video/gstvideoaggregator.h +++ b/gst-libs/gst/video/gstvideoaggregator.h @@ -42,7 +42,7 @@ typedef struct _GstVideoAggregatorPrivate GstVideoAggregatorPrivate; (G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_VIDEO_AGGREGATOR_PAD, GstVideoAggregatorPad)) #define GST_VIDEO_AGGREGATOR_PAD_CAST(obj) ((GstVideoAggregatorPad *)(obj)) #define GST_VIDEO_AGGREGATOR_PAD_CLASS(klass) \ - (G_TYPE_CHECK_CLASS_CAST((klass),GST_TYPE_COMPOSITOR_PAD, GstVideoAggregatorPadClass)) + (G_TYPE_CHECK_CLASS_CAST((klass),GST_TYPE_VIDEO_AGGREGATOR_PAD, GstVideoAggregatorPadClass)) #define GST_IS_VIDEO_AGGREGATOR_PAD(obj) \ (G_TYPE_CHECK_INSTANCE_TYPE((obj),GST_TYPE_VIDEO_AGGREGATOR_PAD)) #define GST_IS_VIDEO_AGGREGATOR_PAD_CLASS(klass) \ @@ -89,10 +89,7 @@ struct _GstVideoAggregatorPad struct _GstVideoAggregatorPadClass { GstAggregatorPadClass parent_class; - gboolean (*set_info) (GstVideoAggregatorPad * pad, - GstVideoAggregator * videoaggregator, - GstVideoInfo * current_info, - GstVideoInfo * wanted_info); + void (*update_conversion_info) (GstVideoAggregatorPad * pad); gboolean (*prepare_frame) (GstVideoAggregatorPad * pad, GstVideoAggregator * videoaggregator, @@ -121,6 +118,59 @@ GstVideoFrame * gst_video_aggregator_pad_get_prepared_frame (GstVideoAggregatorP GST_VIDEO_BAD_API void gst_video_aggregator_pad_set_needs_alpha (GstVideoAggregatorPad *pad, gboolean needs_alpha); +/**************************** + * GstVideoAggregatorPad Structs * + ***************************/ + + +#define GST_TYPE_VIDEO_AGGREGATOR_CONVERT_PAD (gst_video_aggregator_convert_pad_get_type()) +#define GST_VIDEO_AGGREGATOR_CONVERT_PAD(obj) (G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_VIDEO_AGGREGATOR_CONVERT_PAD, GstVideoAggregatorConvertPad)) +#define GST_VIDEO_AGGREGATOR_CONVERT_PAD_CLASS(klass) (G_TYPE_CHECK_CLASS_CAST((klass),GST_TYPE_VIDEO_AGGREGATOR_CONVERT_PAD, GstVideoAggregatorConvertPadClass)) +#define GST_VIDEO_AGGREGATOR_CONVERT_PAD_GET_CLASS(obj) (G_TYPE_INSTANCE_GET_CLASS ((obj),GST_TYPE_VIDEO_AGGREGATOR_CONVERT_PAD, GstVideoAggregatorConvertPadClass)) +#define GST_IS_VIDEO_AGGREGATOR_CONVERT_PAD(obj) (G_TYPE_CHECK_INSTANCE_TYPE((obj),GST_TYPE_VIDEO_AGGREGATOR_CONVERT_PAD)) +#define GST_IS_VIDEO_AGGREGATOR_CONVERT_PAD_CLASS(klass) (G_TYPE_CHECK_CLASS_TYPE((klass),GST_TYPE_VIDEO_AGGREGATOR_CONVERT_PAD)) + +typedef struct _GstVideoAggregatorConvertPad GstVideoAggregatorConvertPad; +typedef struct _GstVideoAggregatorConvertPadClass GstVideoAggregatorConvertPadClass; +typedef struct _GstVideoAggregatorConvertPadPrivate GstVideoAggregatorConvertPadPrivate; + +/** + * GstVideoAggregatorConvertPad: + * + * An implementation of GstPad that can be used with #GstVideoAggregator. + * + * See #GstVideoAggregator for more details. + */ +struct _GstVideoAggregatorConvertPad +{ + /*< private >*/ + GstVideoAggregatorPad parent; + + GstVideoAggregatorConvertPadPrivate *priv; + + gpointer _gst_reserved[GST_PADDING]; +}; + +/** + * GstVideoAggregatorConvertPadClass: + * + */ +struct _GstVideoAggregatorConvertPadClass +{ + GstVideoAggregatorPadClass parent_class; + + void (*create_conversion_info) (GstVideoAggregatorConvertPad *pad, GstVideoAggregator *agg, GstVideoInfo *conversion_info); + + /*< private >*/ + gpointer _gst_reserved[GST_PADDING]; +}; + +GST_VIDEO_BAD_API +GType gst_video_aggregator_convert_pad_get_type (void); + +GST_VIDEO_BAD_API +void gst_video_aggregator_convert_pad_update_conversion_info (GstVideoAggregatorConvertPad * pad); + #define GST_TYPE_VIDEO_AGGREGATOR (gst_video_aggregator_get_type()) #define GST_VIDEO_AGGREGATOR(obj) \ (G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_VIDEO_AGGREGATOR, GstVideoAggregator)) diff --git a/gst/compositor/compositor.c b/gst/compositor/compositor.c index 2c5b76bdb6..341dc70fa3 100644 --- a/gst/compositor/compositor.c +++ b/gst/compositor/compositor.c @@ -141,7 +141,7 @@ enum }; G_DEFINE_TYPE (GstCompositorPad, gst_compositor_pad, - GST_TYPE_VIDEO_AGGREGATOR_PAD); + GST_TYPE_VIDEO_AGGREGATOR_CONVERT_PAD); static void gst_compositor_pad_get_property (GObject * object, guint prop_id, @@ -158,9 +158,13 @@ gst_compositor_pad_get_property (GObject * object, guint prop_id, break; case PROP_PAD_WIDTH: g_value_set_int (value, pad->width); + gst_video_aggregator_convert_pad_update_conversion_info + (GST_VIDEO_AGGREGATOR_CONVERT_PAD (pad)); break; case PROP_PAD_HEIGHT: g_value_set_int (value, pad->height); + gst_video_aggregator_convert_pad_update_conversion_info + (GST_VIDEO_AGGREGATOR_CONVERT_PAD (pad)); break; case PROP_PAD_ALPHA: g_value_set_double (value, pad->alpha); @@ -255,95 +259,6 @@ _mixer_pad_get_output_size (GstCompositor * comp, *height = pad_height; } -static gboolean -gst_compositor_pad_set_info (GstVideoAggregatorPad * pad, - GstVideoAggregator * vagg G_GNUC_UNUSED, - GstVideoInfo * current_info, GstVideoInfo * wanted_info) -{ - GstCompositor *comp = GST_COMPOSITOR (vagg); - GstCompositorPad *cpad = GST_COMPOSITOR_PAD (pad); - gchar *colorimetry, *best_colorimetry; - const gchar *chroma, *best_chroma; - gint width, height; - - if (!current_info->finfo) - return TRUE; - - if (GST_VIDEO_INFO_FORMAT (current_info) == GST_VIDEO_FORMAT_UNKNOWN) - return TRUE; - - if (cpad->convert) - gst_video_converter_free (cpad->convert); - - cpad->convert = NULL; - - if (GST_VIDEO_INFO_MULTIVIEW_MODE (current_info) != - GST_VIDEO_MULTIVIEW_MODE_NONE - && GST_VIDEO_INFO_MULTIVIEW_MODE (current_info) != - GST_VIDEO_MULTIVIEW_MODE_MONO) { - GST_FIXME_OBJECT (pad, "Multiview support is not implemented yet"); - return FALSE; - } - - colorimetry = gst_video_colorimetry_to_string (&(current_info->colorimetry)); - chroma = gst_video_chroma_to_string (current_info->chroma_site); - - best_colorimetry = - gst_video_colorimetry_to_string (&(wanted_info->colorimetry)); - best_chroma = gst_video_chroma_to_string (wanted_info->chroma_site); - - _mixer_pad_get_output_size (comp, cpad, GST_VIDEO_INFO_PAR_N (&vagg->info), - GST_VIDEO_INFO_PAR_D (&vagg->info), &width, &height); - - if (GST_VIDEO_INFO_FORMAT (wanted_info) != - GST_VIDEO_INFO_FORMAT (current_info) - || g_strcmp0 (colorimetry, best_colorimetry) - || g_strcmp0 (chroma, best_chroma) - || width != current_info->width || height != current_info->height) { - GstVideoInfo tmp_info; - - /* Initialize with the wanted video format and our original width and - * height as we don't want to rescale. Then copy over the wanted - * colorimetry, and chroma-site and our current pixel-aspect-ratio - * and other relevant fields. - */ - gst_video_info_set_format (&tmp_info, GST_VIDEO_INFO_FORMAT (wanted_info), - width, height); - tmp_info.chroma_site = wanted_info->chroma_site; - tmp_info.colorimetry = wanted_info->colorimetry; - tmp_info.par_n = wanted_info->par_n; - tmp_info.par_d = wanted_info->par_d; - tmp_info.fps_n = current_info->fps_n; - tmp_info.fps_d = current_info->fps_d; - tmp_info.flags = current_info->flags; - tmp_info.interlace_mode = current_info->interlace_mode; - - GST_DEBUG_OBJECT (pad, "This pad will be converted from format %s to %s, " - "colorimetry %s to %s, chroma-site %s to %s, " - "width/height %d/%d to %d/%d", - current_info->finfo->name, tmp_info.finfo->name, - colorimetry, best_colorimetry, - chroma, best_chroma, - current_info->width, current_info->height, width, height); - - cpad->convert = gst_video_converter_new (current_info, &tmp_info, NULL); - cpad->conversion_info = tmp_info; - if (!cpad->convert) { - g_free (colorimetry); - g_free (best_colorimetry); - GST_WARNING_OBJECT (pad, "No path found for conversion"); - return FALSE; - } - } else { - cpad->conversion_info = *current_info; - GST_DEBUG_OBJECT (pad, "This pad will not need conversion"); - } - g_free (colorimetry); - g_free (best_colorimetry); - - return TRUE; -} - /* Test whether rectangle2 contains rectangle 1 (geometrically) */ static gboolean is_rectangle_contained (GstVideoRectangle rect1, GstVideoRectangle rect2) @@ -382,9 +297,6 @@ gst_compositor_pad_prepare_frame (GstVideoAggregatorPad * pad, { GstCompositor *comp = GST_COMPOSITOR (vagg); GstCompositorPad *cpad = GST_COMPOSITOR_PAD (pad); - guint outsize; - GstVideoFrame frame; - static GstAllocationParams params = { 0, 15, 0, 0, }; gint width, height; gboolean frame_obscured = FALSE; GList *l; @@ -407,70 +319,6 @@ gst_compositor_pad_prepare_frame (GstVideoAggregatorPad * pad, _mixer_pad_get_output_size (comp, cpad, GST_VIDEO_INFO_PAR_N (&vagg->info), GST_VIDEO_INFO_PAR_D (&vagg->info), &width, &height); - /* The only thing that can change here is the width - * and height, otherwise set_info would've been called */ - if (GST_VIDEO_INFO_WIDTH (&cpad->conversion_info) != width || - GST_VIDEO_INFO_HEIGHT (&cpad->conversion_info) != height) { - gchar *colorimetry, *wanted_colorimetry; - const gchar *chroma, *wanted_chroma; - - /* We might end up with no converter afterwards if - * the only reason for conversion was a different - * width or height - */ - if (cpad->convert) - gst_video_converter_free (cpad->convert); - cpad->convert = NULL; - - colorimetry = gst_video_colorimetry_to_string (&pad->info.colorimetry); - chroma = gst_video_chroma_to_string (pad->info.chroma_site); - - wanted_colorimetry = - gst_video_colorimetry_to_string (&cpad->conversion_info.colorimetry); - wanted_chroma = - gst_video_chroma_to_string (cpad->conversion_info.chroma_site); - - if (GST_VIDEO_INFO_FORMAT (&pad->info) != - GST_VIDEO_INFO_FORMAT (&cpad->conversion_info) - || g_strcmp0 (colorimetry, wanted_colorimetry) - || g_strcmp0 (chroma, wanted_chroma) - || width != GST_VIDEO_INFO_WIDTH (&pad->info) - || height != GST_VIDEO_INFO_HEIGHT (&pad->info)) { - GstVideoInfo tmp_info; - - gst_video_info_set_format (&tmp_info, cpad->conversion_info.finfo->format, - width, height); - tmp_info.chroma_site = cpad->conversion_info.chroma_site; - tmp_info.colorimetry = cpad->conversion_info.colorimetry; - tmp_info.par_n = vagg->info.par_n; - tmp_info.par_d = vagg->info.par_d; - tmp_info.fps_n = cpad->conversion_info.fps_n; - tmp_info.fps_d = cpad->conversion_info.fps_d; - tmp_info.flags = cpad->conversion_info.flags; - tmp_info.interlace_mode = cpad->conversion_info.interlace_mode; - - GST_DEBUG_OBJECT (pad, "This pad will be converted from %d to %d", - GST_VIDEO_INFO_FORMAT (&pad->info), - GST_VIDEO_INFO_FORMAT (&tmp_info)); - - cpad->convert = gst_video_converter_new (&pad->info, &tmp_info, NULL); - cpad->conversion_info = tmp_info; - - if (!cpad->convert) { - GST_WARNING_OBJECT (pad, "No path found for conversion"); - g_free (colorimetry); - g_free (wanted_colorimetry); - return FALSE; - } - } else { - GST_VIDEO_INFO_WIDTH (&cpad->conversion_info) = width; - GST_VIDEO_INFO_HEIGHT (&cpad->conversion_info) = height; - } - - g_free (colorimetry); - g_free (wanted_colorimetry); - } - if (cpad->alpha == 0.0) { GST_DEBUG_OBJECT (vagg, "Pad has alpha 0.0, not converting frame"); goto done; @@ -536,37 +384,10 @@ gst_compositor_pad_prepare_frame (GstVideoAggregatorPad * pad, if (frame_obscured) goto done; - if (!gst_video_frame_map (&frame, &pad->info, buffer, GST_MAP_READ)) { - GST_WARNING_OBJECT (vagg, "Could not map input buffer"); - return FALSE; - } - - if (cpad->convert) { - gint converted_size; - GstVideoFrame converted_frame; - GstBuffer *converted_buf = NULL; - - /* We wait until here to set the conversion infos, in case vagg->info changed */ - converted_size = GST_VIDEO_INFO_SIZE (&cpad->conversion_info); - outsize = GST_VIDEO_INFO_SIZE (&vagg->info); - converted_size = converted_size > outsize ? converted_size : outsize; - converted_buf = gst_buffer_new_allocate (NULL, converted_size, ¶ms); - - if (!gst_video_frame_map (&converted_frame, &(cpad->conversion_info), - converted_buf, GST_MAP_READWRITE)) { - GST_WARNING_OBJECT (vagg, "Could not map converted frame"); - - gst_video_frame_unmap (&frame); - return FALSE; - } - - gst_video_converter_frame (cpad->convert, &frame, &converted_frame); - cpad->converted_buffer = converted_buf; - gst_video_frame_unmap (&frame); - *prepared_frame = converted_frame; - } else { - *prepared_frame = frame; - } + return + GST_VIDEO_AGGREGATOR_PAD_CLASS + (gst_compositor_pad_parent_class)->prepare_frame (pad, vagg, buffer, + prepared_frame); done: @@ -574,32 +395,46 @@ done: } static void -gst_compositor_pad_clean_frame (GstVideoAggregatorPad * pad, - GstVideoAggregator * vagg, GstVideoFrame * prepared_frame) +gst_compositor_pad_create_conversion_info (GstVideoAggregatorConvertPad * pad, + GstVideoAggregator * vagg, GstVideoInfo * conversion_info) { + GstCompositor *comp = GST_COMPOSITOR (vagg); GstCompositorPad *cpad = GST_COMPOSITOR_PAD (pad); + gint width, height; - if (prepared_frame->buffer) { - gst_video_frame_unmap (prepared_frame); - memset (prepared_frame, 0, sizeof (GstVideoFrame)); + GST_VIDEO_AGGREGATOR_CONVERT_PAD_CLASS + (gst_compositor_pad_parent_class)->create_conversion_info (pad, vagg, + conversion_info); + if (!conversion_info->finfo) + return; + + _mixer_pad_get_output_size (comp, cpad, GST_VIDEO_INFO_PAR_N (&vagg->info), + GST_VIDEO_INFO_PAR_D (&vagg->info), &width, &height); + + /* The only thing that can change here is the width + * and height, otherwise set_info would've been called */ + if (GST_VIDEO_INFO_WIDTH (conversion_info) != width || + GST_VIDEO_INFO_HEIGHT (conversion_info) != height) { + GstVideoInfo tmp_info; + + /* Initialize with the wanted video format and our original width and + * height as we don't want to rescale. Then copy over the wanted + * colorimetry, and chroma-site and our current pixel-aspect-ratio + * and other relevant fields. + */ + gst_video_info_set_format (&tmp_info, + GST_VIDEO_INFO_FORMAT (conversion_info), width, height); + tmp_info.chroma_site = conversion_info->chroma_site; + tmp_info.colorimetry = conversion_info->colorimetry; + tmp_info.par_n = conversion_info->par_n; + tmp_info.par_d = conversion_info->par_d; + tmp_info.fps_n = conversion_info->fps_n; + tmp_info.fps_d = conversion_info->fps_d; + tmp_info.flags = conversion_info->flags; + tmp_info.interlace_mode = conversion_info->interlace_mode; + + *conversion_info = tmp_info; } - - if (cpad->converted_buffer) { - gst_buffer_unref (cpad->converted_buffer); - cpad->converted_buffer = NULL; - } -} - -static void -gst_compositor_pad_finalize (GObject * object) -{ - GstCompositorPad *pad = GST_COMPOSITOR_PAD (object); - - if (pad->convert) - gst_video_converter_free (pad->convert); - pad->convert = NULL; - - G_OBJECT_CLASS (gst_compositor_pad_parent_class)->finalize (object); } static void @@ -608,10 +443,11 @@ gst_compositor_pad_class_init (GstCompositorPadClass * klass) GObjectClass *gobject_class = (GObjectClass *) klass; GstVideoAggregatorPadClass *vaggpadclass = (GstVideoAggregatorPadClass *) klass; + GstVideoAggregatorConvertPadClass *vaggcpadclass = + (GstVideoAggregatorConvertPadClass *) klass; gobject_class->set_property = gst_compositor_pad_set_property; gobject_class->get_property = gst_compositor_pad_get_property; - gobject_class->finalize = gst_compositor_pad_finalize; g_object_class_install_property (gobject_class, PROP_PAD_XPOS, g_param_spec_int ("xpos", "X Position", "X Position of the picture", @@ -640,11 +476,11 @@ gst_compositor_pad_class_init (GstCompositorPadClass * klass) -1.0, 1.0, DEFAULT_PAD_CROSSFADE_RATIO, G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); - vaggpadclass->set_info = GST_DEBUG_FUNCPTR (gst_compositor_pad_set_info); vaggpadclass->prepare_frame = GST_DEBUG_FUNCPTR (gst_compositor_pad_prepare_frame); - vaggpadclass->clean_frame = - GST_DEBUG_FUNCPTR (gst_compositor_pad_clean_frame); + + vaggcpadclass->create_conversion_info = + GST_DEBUG_FUNCPTR (gst_compositor_pad_create_conversion_info); } static void @@ -1052,6 +888,8 @@ gst_compositor_crossfade_frames (GstCompositor * self, GstVideoFrame * outframe) for (l = GST_ELEMENT (self)->sinkpads; l; l = l->next) { GstVideoAggregatorPad *pad = l->data; GstCompositorPad *compo_pad = GST_COMPOSITOR_PAD (pad); + GstVideoAggregatorPadClass *pad_class = + GST_VIDEO_AGGREGATOR_PAD_GET_CLASS (compo_pad); GstVideoFrame *prepared_frame = gst_video_aggregator_pad_get_prepared_frame (pad); @@ -1084,16 +922,16 @@ gst_compositor_crossfade_frames (GstCompositor * self, GstVideoFrame * outframe) COMPOSITOR_BLEND_MODE_ADDITIVE); /* Replace frame with current frame */ - gst_compositor_pad_clean_frame (npad, vagg, next_prepared_frame); + pad_class->clean_frame (npad, vagg, next_prepared_frame); if (!all_crossfading) *next_prepared_frame = nframe; next_compo_pad->crossfaded = TRUE; /* Frame is now consumed, clean it up */ - gst_compositor_pad_clean_frame (pad, vagg, prepared_frame); + pad_class->clean_frame (pad, vagg, prepared_frame); } else { GST_LOG_OBJECT (self, "Simply fading out as no following pad found"); - gst_compositor_pad_clean_frame (pad, vagg, prepared_frame); + pad_class->clean_frame (pad, vagg, prepared_frame); if (!all_crossfading) *prepared_frame = nframe; compo_pad->crossfaded = TRUE; diff --git a/gst/compositor/compositorpad.h b/gst/compositor/compositorpad.h index 1c2a271a80..88be0e25ab 100644 --- a/gst/compositor/compositorpad.h +++ b/gst/compositor/compositorpad.h @@ -46,7 +46,7 @@ typedef struct _GstCompositorPadClass GstCompositorPadClass; */ struct _GstCompositorPad { - GstVideoAggregatorPad parent; + GstVideoAggregatorConvertPad parent; /* properties */ gint xpos, ypos; @@ -54,16 +54,12 @@ struct _GstCompositorPad gdouble alpha; gdouble crossfade; - GstVideoConverter *convert; - GstVideoInfo conversion_info; - GstBuffer *converted_buffer; - gboolean crossfaded; }; struct _GstCompositorPadClass { - GstVideoAggregatorPadClass parent_class; + GstVideoAggregatorConvertPadClass parent_class; }; GType gst_compositor_pad_get_type (void); From 429e6c85320472a4ec917b638d4463f3af29e78b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Sun, 6 May 2018 15:49:36 +0200 Subject: [PATCH 347/381] videoaggregator: Validate pool configuration and create a new pool if it just does not work Also pass the given allocator to the pool if one is set. --- gst-libs/gst/video/gstvideoaggregator.c | 47 +++++++++++++++++++++---- 1 file changed, 41 insertions(+), 6 deletions(-) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index eb86b4c70c..0aafb31ad6 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -2186,14 +2186,15 @@ gst_video_aggregator_decide_allocation (GstAggregator * agg, GstQuery * query) GstAllocationParams params = { 0, 15, 0, 0 }; guint i; GstBufferPool *pool; + GstAllocator *allocator; guint size, min, max; gboolean update = FALSE; GstStructure *config = NULL; GstCaps *caps = NULL; - if (gst_query_get_n_allocation_params (query) == 0) + if (gst_query_get_n_allocation_params (query) == 0) { gst_query_add_allocation_param (query, NULL, ¶ms); - else + } else { for (i = 0; i < gst_query_get_n_allocation_params (query); i++) { GstAllocator *allocator; @@ -2201,6 +2202,9 @@ gst_video_aggregator_decide_allocation (GstAggregator * agg, GstQuery * query) params.align = MAX (params.align, 15); gst_query_set_nth_allocation_param (query, i, allocator, ¶ms); } + } + + gst_query_parse_nth_allocation_param (query, 0, &allocator, ¶ms); if (gst_query_get_n_allocation_pools (query) > 0) { gst_query_parse_nth_allocation_pool (query, 0, &pool, &size, &min, &max); @@ -2215,21 +2219,39 @@ gst_video_aggregator_decide_allocation (GstAggregator * agg, GstQuery * query) update = FALSE; } + gst_query_parse_allocation (query, &caps, NULL); + /* no downstream pool, make our own */ if (pool == NULL) pool = gst_video_buffer_pool_new (); config = gst_buffer_pool_get_config (pool); - gst_query_parse_allocation (query, &caps, NULL); - if (caps) - gst_buffer_pool_config_set_params (config, caps, size, min, max); + gst_buffer_pool_config_set_params (config, caps, size, min, max); + gst_buffer_pool_config_set_allocator (config, allocator, ¶ms); + + /* buffer pool may have to do some changes */ + if (!gst_buffer_pool_set_config (pool, config)) { + config = gst_buffer_pool_get_config (pool); + + /* If change are not acceptable, fallback to generic pool */ + if (!gst_buffer_pool_config_validate_params (config, caps, size, min, max)) { + GST_DEBUG_OBJECT (agg, "unsupported pool, making new pool"); + + gst_object_unref (pool); + pool = gst_video_buffer_pool_new (); + gst_buffer_pool_config_set_params (config, caps, size, min, max); + gst_buffer_pool_config_set_allocator (config, allocator, ¶ms); + } + + if (!gst_buffer_pool_set_config (pool, config)) + goto config_failed; + } if (gst_query_find_allocation_meta (query, GST_VIDEO_META_API_TYPE, NULL)) { gst_buffer_pool_config_add_option (config, GST_BUFFER_POOL_OPTION_VIDEO_META); } - gst_buffer_pool_set_config (pool, config); if (update) gst_query_set_nth_allocation_pool (query, 0, pool, size, min, max); @@ -2238,8 +2260,21 @@ gst_video_aggregator_decide_allocation (GstAggregator * agg, GstQuery * query) if (pool) gst_object_unref (pool); + if (allocator) + gst_object_unref (allocator); return TRUE; + +config_failed: + if (pool) + gst_object_unref (pool); + if (allocator) + gst_object_unref (allocator); + + GST_ELEMENT_ERROR (agg, RESOURCE, SETTINGS, + ("Failed to configure the buffer pool"), + ("Configuration is most likely invalid, please report this issue.")); + return FALSE; } static GstFlowReturn From dbb21615c47a58389fdc18c1614d5edcfdade082 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Sun, 6 May 2018 16:05:28 +0200 Subject: [PATCH 348/381] videoaggregator: Rename get_output_buffer() to create_output_buffer() For consistency with GstAudioAggregator. --- ext/gl/gstglstereomix.c | 10 +++++----- gst-libs/gst/video/gstvideoaggregator.c | 8 ++++---- gst-libs/gst/video/gstvideoaggregator.h | 4 ++-- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/ext/gl/gstglstereomix.c b/ext/gl/gstglstereomix.c index e9037beaf5..e2bf105d95 100644 --- a/ext/gl/gstglstereomix.c +++ b/ext/gl/gstglstereomix.c @@ -142,8 +142,8 @@ static GstPad *gst_gl_stereo_mix_request_new_pad (GstElement * element, GstPadTemplate * temp, const gchar * req_name, const GstCaps * caps); static void gst_gl_stereo_mix_release_pad (GstElement * element, GstPad * pad); -static GstFlowReturn gst_gl_stereo_mix_get_output_buffer (GstVideoAggregator * - videoaggregator, GstBuffer ** outbuf); +static GstFlowReturn gst_gl_stereo_mix_create_output_buffer (GstVideoAggregator + * videoaggregator, GstBuffer ** outbuf); static gboolean gst_gl_stereo_mix_stop (GstAggregator * agg); static gboolean gst_gl_stereo_mix_start (GstAggregator * agg); static gboolean gst_gl_stereo_mix_src_query (GstAggregator * agg, @@ -205,8 +205,8 @@ gst_gl_stereo_mix_class_init (GstGLStereoMixClass * klass) videoaggregator_class->aggregate_frames = gst_gl_stereo_mix_aggregate_frames; videoaggregator_class->update_caps = _update_caps; - videoaggregator_class->get_output_buffer = - gst_gl_stereo_mix_get_output_buffer; + videoaggregator_class->create_output_buffer = + gst_gl_stereo_mix_create_output_buffer; base_mix_class->supported_gl_api = GST_GL_API_GLES2 | GST_GL_API_OPENGL | GST_GL_API_OPENGL3; @@ -263,7 +263,7 @@ gst_gl_stereo_mix_src_query (GstAggregator * agg, GstQuery * query) static GstFlowReturn -gst_gl_stereo_mix_get_output_buffer (GstVideoAggregator * videoaggregator, +gst_gl_stereo_mix_create_output_buffer (GstVideoAggregator * videoaggregator, GstBuffer ** outbuf) { GstGLStereoMix *mix = GST_GL_STEREO_MIX (videoaggregator); diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index 0aafb31ad6..5d4a3af41e 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -1582,9 +1582,9 @@ gst_video_aggregator_do_aggregate (GstVideoAggregator * vagg, GstVideoAggregatorClass *vagg_klass = (GstVideoAggregatorClass *) klass; g_assert (vagg_klass->aggregate_frames != NULL); - g_assert (vagg_klass->get_output_buffer != NULL); + g_assert (vagg_klass->create_output_buffer != NULL); - if ((ret = vagg_klass->get_output_buffer (vagg, outbuf)) != GST_FLOW_OK) { + if ((ret = vagg_klass->create_output_buffer (vagg, outbuf)) != GST_FLOW_OK) { GST_WARNING_OBJECT (vagg, "Could not get an output buffer, reason: %s", gst_flow_get_name (ret)); return ret; @@ -2278,7 +2278,7 @@ config_failed: } static GstFlowReturn -gst_video_aggregator_get_output_buffer (GstVideoAggregator * videoaggregator, +gst_video_aggregator_create_output_buffer (GstVideoAggregator * videoaggregator, GstBuffer ** outbuf) { GstAggregator *aggregator = GST_AGGREGATOR (videoaggregator); @@ -2499,7 +2499,7 @@ gst_video_aggregator_class_init (GstVideoAggregatorClass * klass) agg_class->propose_allocation = gst_video_aggregator_propose_allocation; klass->find_best_format = gst_video_aggregator_find_best_format; - klass->get_output_buffer = gst_video_aggregator_get_output_buffer; + klass->create_output_buffer = gst_video_aggregator_create_output_buffer; klass->update_caps = gst_video_aggregator_default_update_caps; /* Register the pad class */ diff --git a/gst-libs/gst/video/gstvideoaggregator.h b/gst-libs/gst/video/gstvideoaggregator.h index 611bd9c56a..5926660a16 100644 --- a/gst-libs/gst/video/gstvideoaggregator.h +++ b/gst-libs/gst/video/gstvideoaggregator.h @@ -213,7 +213,7 @@ struct _GstVideoAggregator * or directly use the #GstBuffer from GstVideoAggregatorPad.buffer * if it needs to map the buffer in a special way. The result of the * aggregation should land in @outbuffer. - * @get_output_buffer: Optional. + * @create_output_buffer: Optional. * Lets subclasses provide a #GstBuffer to be used as @outbuffer of * the #aggregate_frames vmethod. * @negotiated_caps: Optional. @@ -231,7 +231,7 @@ struct _GstVideoAggregatorClass GstCaps * caps); GstFlowReturn (*aggregate_frames) (GstVideoAggregator * videoaggregator, GstBuffer * outbuffer); - GstFlowReturn (*get_output_buffer) (GstVideoAggregator * videoaggregator, + GstFlowReturn (*create_output_buffer) (GstVideoAggregator * videoaggregator, GstBuffer ** outbuffer); void (*find_best_format) (GstVideoAggregator * vagg, GstCaps * downstream_caps, From 9d6d46f4cb5c1eb93f2353ce917d1f3945ce76e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Sun, 6 May 2018 16:22:01 +0200 Subject: [PATCH 349/381] videoaggregator: Clean up header and update docs a bit --- gst-libs/gst/video/gstvideoaggregator.c | 1 - gst-libs/gst/video/gstvideoaggregator.h | 52 ++++++++++++------------- 2 files changed, 26 insertions(+), 27 deletions(-) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index 5d4a3af41e..7d9bfec9ac 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -313,7 +313,6 @@ gst_video_aggregator_pad_get_prepared_frame (GstVideoAggregatorPad * pad) * * Allows selecting that this pad requires an output format with alpha * - * Returns: (transfer none): The currently prepared video frame */ void gst_video_aggregator_pad_set_needs_alpha (GstVideoAggregatorPad * pad, diff --git a/gst-libs/gst/video/gstvideoaggregator.h b/gst-libs/gst/video/gstvideoaggregator.h index 5926660a16..6f6fccad3a 100644 --- a/gst-libs/gst/video/gstvideoaggregator.h +++ b/gst-libs/gst/video/gstvideoaggregator.h @@ -37,6 +37,10 @@ typedef struct _GstVideoAggregator GstVideoAggregator; typedef struct _GstVideoAggregatorClass GstVideoAggregatorClass; typedef struct _GstVideoAggregatorPrivate GstVideoAggregatorPrivate; +/************************* + * GstVideoAggregatorPad * + *************************/ + #define GST_TYPE_VIDEO_AGGREGATOR_PAD (gst_video_aggregator_pad_get_type()) #define GST_VIDEO_AGGREGATOR_PAD(obj) \ (G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_VIDEO_AGGREGATOR_PAD, GstVideoAggregatorPad)) @@ -57,31 +61,26 @@ typedef struct _GstVideoAggregatorPadPrivate GstVideoAggregatorPadPrivate; /** * GstVideoAggregatorPad: * @info: The #GstVideoInfo currently set on the pad - * @buffer_vinfo: The #GstVideoInfo representing the type contained - * in @buffer - * @aggregated_frame: The #GstVideoFrame ready to be used for aggregation - * inside the aggregate_frames vmethod. - * @zorder: The zorder of this pad */ struct _GstVideoAggregatorPad { - GstAggregatorPad parent; + GstAggregatorPad parent; /*< public >*/ /* read-only, with OBJECT_LOCK */ - GstVideoInfo info; + GstVideoInfo info; /* < private > */ GstVideoAggregatorPadPrivate *priv; - gpointer _gst_reserved[GST_PADDING]; + gpointer _gst_reserved[GST_PADDING]; }; /** * GstVideoAggregatorPadClass: * - * @set_info: Lets subclass set a converter on the pad, - * right after a new format has been negotiated. + * @update_conversion_info: Called when either the input or output formats + * have changed. * @prepare_frame: Prepare the frame from the pad buffer (if any) * and sets it to @aggregated_frame * @clean_frame: clean the frame previously prepared in prepare_frame @@ -91,14 +90,14 @@ struct _GstVideoAggregatorPadClass GstAggregatorPadClass parent_class; void (*update_conversion_info) (GstVideoAggregatorPad * pad); - gboolean (*prepare_frame) (GstVideoAggregatorPad * pad, - GstVideoAggregator * videoaggregator, - GstBuffer * buffer, - GstVideoFrame * prepared_frame); + gboolean (*prepare_frame) (GstVideoAggregatorPad * pad, + GstVideoAggregator * videoaggregator, + GstBuffer * buffer, + GstVideoFrame * prepared_frame); - void (*clean_frame) (GstVideoAggregatorPad * pad, - GstVideoAggregator * videoaggregator, - GstVideoFrame * prepared_frame); + void (*clean_frame) (GstVideoAggregatorPad * pad, + GstVideoAggregator * videoaggregator, + GstVideoFrame * prepared_frame); gpointer _gst_reserved[GST_PADDING_LARGE]; }; @@ -118,10 +117,9 @@ GstVideoFrame * gst_video_aggregator_pad_get_prepared_frame (GstVideoAggregatorP GST_VIDEO_BAD_API void gst_video_aggregator_pad_set_needs_alpha (GstVideoAggregatorPad *pad, gboolean needs_alpha); -/**************************** - * GstVideoAggregatorPad Structs * - ***************************/ - +/******************************** + * GstVideoAggregatorConvertPad * + *******************************/ #define GST_TYPE_VIDEO_AGGREGATOR_CONVERT_PAD (gst_video_aggregator_convert_pad_get_type()) #define GST_VIDEO_AGGREGATOR_CONVERT_PAD(obj) (G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_VIDEO_AGGREGATOR_CONVERT_PAD, GstVideoAggregatorConvertPad)) @@ -171,6 +169,10 @@ GType gst_video_aggregator_convert_pad_get_type (void); GST_VIDEO_BAD_API void gst_video_aggregator_convert_pad_update_conversion_info (GstVideoAggregatorConvertPad * pad); +/********************** + * GstVideoAggregator * + *********************/ + #define GST_TYPE_VIDEO_AGGREGATOR (gst_video_aggregator_get_type()) #define GST_VIDEO_AGGREGATOR(obj) \ (G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_VIDEO_AGGREGATOR, GstVideoAggregator)) @@ -191,15 +193,15 @@ void gst_video_aggregator_convert_pad_update_conversion_info (GstVideoAggregator */ struct _GstVideoAggregator { - GstAggregator aggregator; + GstAggregator aggregator; /*< public >*/ /* Output caps */ - GstVideoInfo info; + GstVideoInfo info; /* < private > */ GstVideoAggregatorPrivate *priv; - gpointer _gst_reserved[GST_PADDING_LARGE]; + gpointer _gst_reserved[GST_PADDING_LARGE]; }; /** @@ -216,8 +218,6 @@ struct _GstVideoAggregator * @create_output_buffer: Optional. * Lets subclasses provide a #GstBuffer to be used as @outbuffer of * the #aggregate_frames vmethod. - * @negotiated_caps: Optional. - * Notifies subclasses what caps format has been negotiated * @find_best_format: Optional. * Lets subclasses decide of the best common format to use. **/ From 049877e5d0258fbadbe1e2ddea0faa71234ca1da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Sun, 6 May 2018 16:43:32 +0200 Subject: [PATCH 350/381] videoaggregator: Fix up documentation some more --- gst-libs/gst/video/gstvideoaggregator.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index 7d9bfec9ac..c6fbf541ba 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -261,7 +261,7 @@ gst_video_aggregator_pad_has_current_buffer (GstVideoAggregatorPad * pad) } /** - * gst_video_aggregator_pad_has_current_buffer: + * gst_video_aggregator_pad_get_current_buffer: * @pad: a #GstVideoAggregatorPad * * Returns the currently queued buffer that is going to be used From 1492fb4ef93fe076a8b91f19c2b956a2bb60de0c Mon Sep 17 00:00:00 2001 From: Mathieu Duponchelle Date: Sun, 6 May 2018 16:44:47 +0200 Subject: [PATCH 351/381] videoaggregator: expose converter-config on convert pads This in order to allow users control over the conversion process, for example the scaling method. --- gst-libs/gst/video/gstvideoaggregator.c | 67 ++++++++++++++++++++++++- 1 file changed, 66 insertions(+), 1 deletion(-) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index c6fbf541ba..ce56543953 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -335,6 +335,12 @@ gst_video_aggregator_pad_set_needs_alpha (GstVideoAggregatorPad * pad, * GstVideoAggregatorConvertPad implementation * ****************************************/ +enum +{ + PROP_CONVERT_PAD_0, + PROP_CONVERT_PAD_CONVERTER_CONFIG, +}; + struct _GstVideoAggregatorConvertPadPrivate { /* Converter, if NULL no conversion is done */ @@ -344,6 +350,7 @@ struct _GstVideoAggregatorConvertPadPrivate GstVideoInfo conversion_info; GstBuffer *converted_buffer; + GstStructure *converter_config; gboolean converter_config_changed; }; @@ -359,6 +366,10 @@ gst_video_aggregator_convert_pad_finalize (GObject * o) gst_video_converter_free (vaggpad->priv->convert); vaggpad->priv->convert = NULL; + if (vaggpad->priv->converter_config) + gst_structure_free (vaggpad->priv->converter_config); + vaggpad->priv->converter_config = NULL; + G_OBJECT_CLASS (gst_video_aggregator_pad_parent_class)->finalize (o); } @@ -403,7 +414,8 @@ gst_video_aggregator_convert_pad_prepare_frame (GstVideoAggregatorPad * vpad, if (!gst_video_info_is_equal (&vpad->info, &pad->priv->conversion_info)) { pad->priv->convert = gst_video_converter_new (&vpad->info, &pad->priv->conversion_info, - NULL); + pad->priv->converter_config ? gst_structure_copy (pad-> + priv->converter_config) : NULL); if (!pad->priv->convert) { GST_WARNING_OBJECT (pad, "No path found for conversion"); return FALSE; @@ -530,6 +542,46 @@ static void g_free (best_colorimetry); } +static void +gst_video_aggregator_convert_pad_get_property (GObject * object, guint prop_id, + GValue * value, GParamSpec * pspec) +{ + GstVideoAggregatorConvertPad *pad = GST_VIDEO_AGGREGATOR_CONVERT_PAD (object); + + switch (prop_id) { + case PROP_CONVERT_PAD_CONVERTER_CONFIG: + GST_OBJECT_LOCK (pad); + if (pad->priv->converter_config) + g_value_set_boxed (value, pad->priv->converter_config); + GST_OBJECT_UNLOCK (pad); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static void +gst_video_aggregator_convert_pad_set_property (GObject * object, guint prop_id, + const GValue * value, GParamSpec * pspec) +{ + GstVideoAggregatorConvertPad *pad = GST_VIDEO_AGGREGATOR_CONVERT_PAD (object); + + switch (prop_id) { + case PROP_CONVERT_PAD_CONVERTER_CONFIG: + GST_OBJECT_LOCK (pad); + if (pad->priv->converter_config) + gst_structure_free (pad->priv->converter_config); + pad->priv->converter_config = g_value_dup_boxed (value); + pad->priv->converter_config_changed = TRUE; + GST_OBJECT_UNLOCK (pad); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + static void gst_video_aggregator_convert_pad_class_init (GstVideoAggregatorConvertPadClass * klass) @@ -543,6 +595,18 @@ gst_video_aggregator_convert_pad_class_init (GstVideoAggregatorConvertPadClass * g_type_class_add_private (klass, sizeof (GstVideoAggregatorConvertPadPrivate)); + g_object_class_install_property (gobject_class, + PROP_CONVERT_PAD_CONVERTER_CONFIG, g_param_spec_boxed ("converter-config", + "Converter configuration", + "A GstStructure describing the configuration that should be used " + "when scaling and converting this pad's video frames", + GST_TYPE_STRUCTURE, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + + gobject_class->get_property = + GST_DEBUG_FUNCPTR (gst_video_aggregator_convert_pad_get_property); + gobject_class->set_property = + GST_DEBUG_FUNCPTR (gst_video_aggregator_convert_pad_set_property); + vaggpadclass->update_conversion_info = GST_DEBUG_FUNCPTR (gst_video_aggregator_convert_pad_update_conversion_info_internal); @@ -565,6 +629,7 @@ gst_video_aggregator_convert_pad_init (GstVideoAggregatorConvertPad * vaggpad) vaggpad->priv->converted_buffer = NULL; vaggpad->priv->convert = NULL; + vaggpad->priv->converter_config = NULL; vaggpad->priv->converter_config_changed = FALSE; } From 73f843539de9129884bc2e233f50fe24f5910f77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Sun, 6 May 2018 16:49:43 +0200 Subject: [PATCH 352/381] videoaggregator: Some more documentation fixes --- gst-libs/gst/video/gstvideoaggregator.h | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/gst-libs/gst/video/gstvideoaggregator.h b/gst-libs/gst/video/gstvideoaggregator.h index 6f6fccad3a..6b53ce41c1 100644 --- a/gst-libs/gst/video/gstvideoaggregator.h +++ b/gst-libs/gst/video/gstvideoaggregator.h @@ -81,8 +81,7 @@ struct _GstVideoAggregatorPad * * @update_conversion_info: Called when either the input or output formats * have changed. - * @prepare_frame: Prepare the frame from the pad buffer (if any) - * and sets it to @aggregated_frame + * @prepare_frame: Prepare the frame from the pad buffer and sets it to prepared_frame * @clean_frame: clean the frame previously prepared in prepare_frame */ struct _GstVideoAggregatorPadClass @@ -211,8 +210,8 @@ struct _GstVideoAggregator * the src pad caps before usage. Return %NULL to indicate failure. * @aggregate_frames: Lets subclasses aggregate frames that are ready. Subclasses * should iterate the GstElement.sinkpads and use the already - * mapped #GstVideoFrame from GstVideoAggregatorPad.aggregated_frame - * or directly use the #GstBuffer from GstVideoAggregatorPad.buffer + * mapped #GstVideoFrame from gst_video_aggregator_pad_get_prepared_frame() + * or directly use the #GstBuffer from gst_video_aggregator_pad_get_current_buffer() * if it needs to map the buffer in a special way. The result of the * aggregation should land in @outbuffer. * @create_output_buffer: Optional. From 4291ee94c4828c061ab1c1677085982908fb32b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Mon, 7 May 2018 09:17:16 +0200 Subject: [PATCH 353/381] videoaggregator: First override set/get_property vfuncs, then install properties Gives assertions otherwise. --- gst-libs/gst/video/gstvideoaggregator.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index ce56543953..240602fb61 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -591,6 +591,10 @@ gst_video_aggregator_convert_pad_class_init (GstVideoAggregatorConvertPadClass * (GstVideoAggregatorPadClass *) klass; gobject_class->finalize = gst_video_aggregator_convert_pad_finalize; + gobject_class->get_property = + GST_DEBUG_FUNCPTR (gst_video_aggregator_convert_pad_get_property); + gobject_class->set_property = + GST_DEBUG_FUNCPTR (gst_video_aggregator_convert_pad_set_property); g_type_class_add_private (klass, sizeof (GstVideoAggregatorConvertPadPrivate)); @@ -602,11 +606,6 @@ gst_video_aggregator_convert_pad_class_init (GstVideoAggregatorConvertPadClass * "when scaling and converting this pad's video frames", GST_TYPE_STRUCTURE, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); - gobject_class->get_property = - GST_DEBUG_FUNCPTR (gst_video_aggregator_convert_pad_get_property); - gobject_class->set_property = - GST_DEBUG_FUNCPTR (gst_video_aggregator_convert_pad_set_property); - vaggpadclass->update_conversion_info = GST_DEBUG_FUNCPTR (gst_video_aggregator_convert_pad_update_conversion_info_internal); From c43bae0a42f2e2f02b45bf1a1e97e162f5ae1916 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Mon, 7 May 2018 17:53:32 +0300 Subject: [PATCH 354/381] videoaggregator: Set video-meta option on buffer pool configuration correctly CID 1435451 --- gst-libs/gst/video/gstvideoaggregator.c | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index 240602fb61..8474f2de6c 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -2292,6 +2292,10 @@ gst_video_aggregator_decide_allocation (GstAggregator * agg, GstQuery * query) gst_buffer_pool_config_set_params (config, caps, size, min, max); gst_buffer_pool_config_set_allocator (config, allocator, ¶ms); + if (gst_query_find_allocation_meta (query, GST_VIDEO_META_API_TYPE, NULL)) { + gst_buffer_pool_config_add_option (config, + GST_BUFFER_POOL_OPTION_VIDEO_META); + } /* buffer pool may have to do some changes */ if (!gst_buffer_pool_set_config (pool, config)) { @@ -2305,17 +2309,17 @@ gst_video_aggregator_decide_allocation (GstAggregator * agg, GstQuery * query) pool = gst_video_buffer_pool_new (); gst_buffer_pool_config_set_params (config, caps, size, min, max); gst_buffer_pool_config_set_allocator (config, allocator, ¶ms); + + if (gst_query_find_allocation_meta (query, GST_VIDEO_META_API_TYPE, NULL)) { + gst_buffer_pool_config_add_option (config, + GST_BUFFER_POOL_OPTION_VIDEO_META); + } } if (!gst_buffer_pool_set_config (pool, config)) goto config_failed; } - if (gst_query_find_allocation_meta (query, GST_VIDEO_META_API_TYPE, NULL)) { - gst_buffer_pool_config_add_option (config, - GST_BUFFER_POOL_OPTION_VIDEO_META); - } - if (update) gst_query_set_nth_allocation_pool (query, 0, pool, size, min, max); else From 91692122d3404cd4dd0accab62df6e00acc868cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Olivier=20Cr=C3=AAte?= Date: Mon, 23 Apr 2018 13:30:38 -0400 Subject: [PATCH 355/381] videoaggregator: Remove custom get_next_time implementation GstAggregator now has the same thing in the simple implementation. https://bugzilla.gnome.org/show_bug.cgi?id=795486 --- gst-libs/gst/video/gstvideoaggregator.c | 24 +----------------------- 1 file changed, 1 insertion(+), 23 deletions(-) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index 8474f2de6c..0848556f26 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -1717,28 +1717,6 @@ gst_video_aggregator_do_qos (GstVideoAggregator * vagg, GstClockTime timestamp) return jitter; } -static GstClockTime -gst_video_aggregator_get_next_time (GstAggregator * agg) -{ - GstClockTime next_time; - GstSegment *agg_segment = &GST_AGGREGATOR_PAD (agg->srcpad)->segment; - - GST_OBJECT_LOCK (agg); - if (agg_segment->position == -1 || agg_segment->position < agg_segment->start) - next_time = agg_segment->start; - else - next_time = agg_segment->position; - - if (agg_segment->stop != -1 && next_time > agg_segment->stop) - next_time = agg_segment->stop; - - next_time = - gst_segment_to_running_time (agg_segment, GST_FORMAT_TIME, next_time); - GST_OBJECT_UNLOCK (agg); - - return next_time; -} - static void gst_video_aggregator_advance_on_timeout (GstVideoAggregator * vagg) { @@ -2557,7 +2535,7 @@ gst_video_aggregator_class_init (GstVideoAggregatorClass * klass) agg_class->aggregate = gst_video_aggregator_aggregate; agg_class->src_event = gst_video_aggregator_src_event; agg_class->src_query = gst_video_aggregator_src_query; - agg_class->get_next_time = gst_video_aggregator_get_next_time; + agg_class->get_next_time = gst_aggregator_simple_get_next_time; agg_class->update_src_caps = gst_video_aggregator_default_update_src_caps; agg_class->fixate_src_caps = gst_video_aggregator_default_fixate_src_caps; agg_class->negotiated_src_caps = From ecd58f27754f209c0ad3c8b0e74ecc990ea0b618 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim-Philipp=20M=C3=BCller?= Date: Mon, 11 Jun 2018 13:48:09 +0100 Subject: [PATCH 356/381] videoaggregator: log an ERROR if we're going to return a flow error --- gst-libs/gst/video/gstvideoaggregator.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index 0848556f26..67de8909f8 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -1402,7 +1402,7 @@ gst_video_aggregator_fill_queues (GstVideoAggregator * vagg, start_time = GST_BUFFER_TIMESTAMP (buf); if (start_time == -1) { gst_buffer_unref (buf); - GST_DEBUG_OBJECT (pad, "Need timestamped buffers!"); + GST_ERROR_OBJECT (pad, "Need timestamped buffers!"); GST_OBJECT_UNLOCK (vagg); return GST_FLOW_ERROR; } From 5934fb1bb626f569a072dc3e6fd9e9a92fbc70a5 Mon Sep 17 00:00:00 2001 From: Seungha Yang Date: Fri, 15 Jun 2018 22:48:42 +0900 Subject: [PATCH 357/381] videoaggregator: Fix string leak gst_video_colorimetry_to_string() returns allocated memory which must be freed. https://bugzilla.gnome.org/show_bug.cgi?id=796596 --- gst-libs/gst/video/gstvideoaggregator.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index 67de8909f8..4e1fc26bfb 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -891,13 +891,14 @@ gst_video_aggregator_default_update_caps (GstVideoAggregator * vagg, gst_caps_unref (tmp); } + color_name = gst_video_colorimetry_to_string (&best_info.colorimetry); + GST_DEBUG_OBJECT (vagg, "The output format will now be : %d with chroma : %s and colorimetry %s", best_format, gst_video_chroma_to_string (best_info.chroma_site), - gst_video_colorimetry_to_string (&best_info.colorimetry)); + color_name); best_format_caps = gst_caps_copy (caps); - color_name = gst_video_colorimetry_to_string (&best_info.colorimetry); gst_caps_set_simple (best_format_caps, "format", G_TYPE_STRING, gst_video_format_to_string (best_format), "chroma-site", G_TYPE_STRING, gst_video_chroma_to_string (best_info.chroma_site), "colorimetry", From 69534fe9a036480c47eeab882fe7e743a8d352f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim-Philipp=20M=C3=BCller?= Date: Sun, 24 Jun 2018 00:17:26 +0200 Subject: [PATCH 358/381] gl: Update for g_type_class_add_private() deprecation in recent GLib --- ext/gl/gstglbasemixer.c | 20 +++++--------------- ext/gl/gstglmixer.c | 14 +++++--------- ext/gl/gstglmixerbin.c | 7 ++----- 3 files changed, 12 insertions(+), 29 deletions(-) diff --git a/ext/gl/gstglbasemixer.c b/ext/gl/gstglbasemixer.c index 0dd3f81695..0284c8dc2b 100644 --- a/ext/gl/gstglbasemixer.c +++ b/ext/gl/gstglbasemixer.c @@ -28,10 +28,6 @@ #include "gstglbasemixer.h" -#define gst_gl_base_mixer_parent_class parent_class -G_DEFINE_ABSTRACT_TYPE (GstGLBaseMixer, gst_gl_base_mixer, - GST_TYPE_VIDEO_AGGREGATOR); - #define GST_CAT_DEFAULT gst_gl_base_mixer_debug GST_DEBUG_CATEGORY (gst_gl_base_mixer_debug); @@ -45,14 +41,6 @@ static void gst_gl_base_mixer_set_context (GstElement * element, static GstStateChangeReturn gst_gl_base_mixer_change_state (GstElement * element, GstStateChange transition); -enum -{ - PROP_PAD_0 -}; - -#define GST_GL_BASE_MIXER_GET_PRIVATE(obj) \ - (G_TYPE_INSTANCE_GET_PRIVATE ((obj), GST_TYPE_GL_BASE_MIXER, GstGLBaseMixerPrivate)) - struct _GstGLBaseMixerPrivate { gboolean negotiated; @@ -60,6 +48,10 @@ struct _GstGLBaseMixerPrivate GstGLContext *other_context; }; +#define gst_gl_base_mixer_parent_class parent_class +G_DEFINE_ABSTRACT_TYPE_WITH_PRIVATE (GstGLBaseMixer, gst_gl_base_mixer, + GST_TYPE_VIDEO_AGGREGATOR); + G_DEFINE_TYPE (GstGLBaseMixerPad, gst_gl_base_mixer_pad, GST_TYPE_VIDEO_AGGREGATOR_PAD); @@ -260,8 +252,6 @@ gst_gl_base_mixer_class_init (GstGLBaseMixerClass * klass) gobject_class = (GObjectClass *) klass; element_class = GST_ELEMENT_CLASS (klass); - g_type_class_add_private (klass, sizeof (GstGLBaseMixerPrivate)); - gobject_class->get_property = gst_gl_base_mixer_get_property; gobject_class->set_property = gst_gl_base_mixer_set_property; @@ -292,7 +282,7 @@ gst_gl_base_mixer_class_init (GstGLBaseMixerClass * klass) static void gst_gl_base_mixer_init (GstGLBaseMixer * mix) { - mix->priv = GST_GL_BASE_MIXER_GET_PRIVATE (mix); + mix->priv = gst_gl_base_mixer_get_instance_private (mix); } static void diff --git a/ext/gl/gstglmixer.c b/ext/gl/gstglmixer.c index fcc95427c7..682622d6d9 100644 --- a/ext/gl/gstglmixer.c +++ b/ext/gl/gstglmixer.c @@ -30,9 +30,6 @@ #include "gstglmixer.h" -#define gst_gl_mixer_parent_class parent_class -G_DEFINE_ABSTRACT_TYPE (GstGLMixer, gst_gl_mixer, GST_TYPE_GL_BASE_MIXER); - #define GST_CAT_DEFAULT gst_gl_mixer_debug GST_DEBUG_CATEGORY (gst_gl_mixer_debug); @@ -51,9 +48,6 @@ enum PROP_PAD_0 }; -#define GST_GL_MIXER_GET_PRIVATE(obj) \ - (G_TYPE_INSTANCE_GET_PRIVATE ((obj), GST_TYPE_GL_MIXER, GstGLMixerPrivate)) - struct _GstGLMixerPrivate { gboolean negotiated; @@ -63,6 +57,10 @@ struct _GstGLMixerPrivate GCond gl_resource_cond; }; +#define gst_gl_mixer_parent_class parent_class +G_DEFINE_ABSTRACT_TYPE_WITH_PRIVATE (GstGLMixer, gst_gl_mixer, + GST_TYPE_GL_BASE_MIXER); + G_DEFINE_TYPE (GstGLMixerPad, gst_gl_mixer_pad, GST_TYPE_GL_BASE_MIXER_PAD); static void @@ -407,8 +405,6 @@ gst_gl_mixer_class_init (GstGLMixerClass * klass) GST_DEBUG_CATEGORY_INIT (GST_CAT_DEFAULT, "glmixer", 0, "OpenGL mixer"); - g_type_class_add_private (klass, sizeof (GstGLMixerPrivate)); - gobject_class->finalize = GST_DEBUG_FUNCPTR (gst_gl_mixer_finalize); gobject_class->get_property = gst_gl_mixer_get_property; @@ -446,7 +442,7 @@ gst_gl_mixer_reset (GstGLMixer * mix) static void gst_gl_mixer_init (GstGLMixer * mix) { - mix->priv = GST_GL_MIXER_GET_PRIVATE (mix); + mix->priv = gst_gl_mixer_get_instance_private (mix); mix->priv->gl_resource_ready = FALSE; g_mutex_init (&mix->priv->gl_resource_lock); diff --git a/ext/gl/gstglmixerbin.c b/ext/gl/gstglmixerbin.c index 1e6ebeb14e..94525e1de0 100644 --- a/ext/gl/gstglmixerbin.c +++ b/ext/gl/gstglmixerbin.c @@ -126,9 +126,8 @@ enum static void gst_gl_mixer_bin_child_proxy_init (gpointer g_iface, gpointer iface_data); -#define GST_GL_MIXER_BIN_GET_PRIVATE(o) \ - (G_TYPE_INSTANCE_GET_PRIVATE((o), GST_TYPE_GL_MIXER_BIN, GstGLMixerBinPrivate)) G_DEFINE_TYPE_WITH_CODE (GstGLMixerBin, gst_gl_mixer_bin, GST_TYPE_BIN, + G_ADD_PRIVATE (GstGLMixerBin) G_IMPLEMENT_INTERFACE (GST_TYPE_CHILD_PROXY, gst_gl_mixer_bin_child_proxy_init)); @@ -159,8 +158,6 @@ gst_gl_mixer_bin_class_init (GstGLMixerBinClass * klass) GstElementClass *element_class = GST_ELEMENT_CLASS (klass); GstCaps *upload_caps; - g_type_class_add_private (klass, sizeof (GstGLMixerBinPrivate)); - GST_DEBUG_CATEGORY_INIT (GST_CAT_DEFAULT, "glmixerbin", 0, "opengl mixer bin"); @@ -233,7 +230,7 @@ gst_gl_mixer_bin_init (GstGLMixerBin * self) gboolean res = TRUE; GstPad *pad; - self->priv = GST_GL_MIXER_BIN_GET_PRIVATE (self); + self->priv = gst_gl_mixer_bin_get_instance_private (self); self->out_convert = gst_element_factory_make ("glcolorconvert", NULL); self->download = gst_element_factory_make ("gldownload", NULL); From b342955a641e1b35aa1ecffc099c5aabb1add745 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim-Philipp=20M=C3=BCller?= Date: Sun, 24 Jun 2018 00:17:26 +0200 Subject: [PATCH 359/381] videoaggregator: Update for g_type_class_add_private() deprecation in recent GLib --- gst-libs/gst/video/gstvideoaggregator.c | 39 +++++++++++++------------ 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index 4e1fc26bfb..dafd59d933 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -83,7 +83,7 @@ struct _GstVideoAggregatorPadPrivate }; -G_DEFINE_TYPE (GstVideoAggregatorPad, gst_video_aggregator_pad, +G_DEFINE_TYPE_WITH_PRIVATE (GstVideoAggregatorPad, gst_video_aggregator_pad, GST_TYPE_AGGREGATOR_PAD); static void @@ -218,8 +218,6 @@ gst_video_aggregator_pad_class_init (GstVideoAggregatorPadClass * klass) DEFAULT_PAD_REPEAT_AFTER_EOS, G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); - g_type_class_add_private (klass, sizeof (GstVideoAggregatorPadPrivate)); - aggpadclass->flush = GST_DEBUG_FUNCPTR (_flush_pad); aggpadclass->skip_buffer = GST_DEBUG_FUNCPTR (gst_video_aggregator_pad_skip_buffer); @@ -231,9 +229,7 @@ gst_video_aggregator_pad_class_init (GstVideoAggregatorPadClass * klass) static void gst_video_aggregator_pad_init (GstVideoAggregatorPad * vaggpad) { - vaggpad->priv = - G_TYPE_INSTANCE_GET_PRIVATE (vaggpad, GST_TYPE_VIDEO_AGGREGATOR_PAD, - GstVideoAggregatorPadPrivate); + vaggpad->priv = gst_video_aggregator_pad_get_instance_private (vaggpad); vaggpad->priv->zorder = DEFAULT_PAD_ZORDER; vaggpad->priv->repeat_after_eos = DEFAULT_PAD_REPEAT_AFTER_EOS; @@ -354,8 +350,8 @@ struct _GstVideoAggregatorConvertPadPrivate gboolean converter_config_changed; }; -G_DEFINE_TYPE (GstVideoAggregatorConvertPad, gst_video_aggregator_convert_pad, - GST_TYPE_VIDEO_AGGREGATOR_PAD); +G_DEFINE_TYPE_WITH_PRIVATE (GstVideoAggregatorConvertPad, + gst_video_aggregator_convert_pad, GST_TYPE_VIDEO_AGGREGATOR_PAD); static void gst_video_aggregator_convert_pad_finalize (GObject * o) @@ -596,9 +592,6 @@ gst_video_aggregator_convert_pad_class_init (GstVideoAggregatorConvertPadClass * gobject_class->set_property = GST_DEBUG_FUNCPTR (gst_video_aggregator_convert_pad_set_property); - g_type_class_add_private (klass, - sizeof (GstVideoAggregatorConvertPadPrivate)); - g_object_class_install_property (gobject_class, PROP_CONVERT_PAD_CONVERTER_CONFIG, g_param_spec_boxed ("converter-config", "Converter configuration", @@ -622,9 +615,7 @@ static void gst_video_aggregator_convert_pad_init (GstVideoAggregatorConvertPad * vaggpad) { vaggpad->priv = - G_TYPE_INSTANCE_GET_PRIVATE (vaggpad, - GST_TYPE_VIDEO_AGGREGATOR_CONVERT_PAD, - GstVideoAggregatorConvertPadPrivate); + gst_video_aggregator_convert_pad_get_instance_private (vaggpad); vaggpad->priv->converted_buffer = NULL; vaggpad->priv->convert = NULL; @@ -700,6 +691,7 @@ static void gst_video_aggregator_init (GstVideoAggregator * self, GstVideoAggregatorClass * klass); static void gst_video_aggregator_class_init (GstVideoAggregatorClass * klass); static gpointer gst_video_aggregator_parent_class = NULL; +static gint video_aggregator_private_offset = 0; GType gst_video_aggregator_get_type (void) @@ -714,11 +706,22 @@ gst_video_aggregator_get_type (void) sizeof (GstVideoAggregator), (GInstanceInitFunc) gst_video_aggregator_init, (GTypeFlags) G_TYPE_FLAG_ABSTRACT); + + video_aggregator_private_offset = + g_type_add_instance_private (g_define_type_id, + sizeof (GstVideoAggregatorPrivate)); + g_once_init_leave (&g_define_type_id_volatile, g_define_type_id); } return g_define_type_id_volatile; } +static inline GstVideoAggregatorPrivate * +gst_video_aggregator_get_instance_private (GstVideoAggregator * self) +{ + return (G_STRUCT_MEMBER_P (self, video_aggregator_private_offset)); +} + static void gst_video_aggregator_find_best_format (GstVideoAggregator * vagg, GstCaps * downstream_caps, GstVideoInfo * best_info, @@ -2515,7 +2518,9 @@ gst_video_aggregator_class_init (GstVideoAggregatorClass * klass) gst_video_aggregator_parent_class = g_type_class_peek_parent (klass); - g_type_class_add_private (klass, sizeof (GstVideoAggregatorPrivate)); + if (video_aggregator_private_offset != 0) + g_type_class_adjust_private_offset (klass, + &video_aggregator_private_offset); gobject_class->finalize = gst_video_aggregator_finalize; gobject_class->dispose = gst_video_aggregator_dispose; @@ -2556,9 +2561,7 @@ static void gst_video_aggregator_init (GstVideoAggregator * vagg, GstVideoAggregatorClass * klass) { - vagg->priv = - G_TYPE_INSTANCE_GET_PRIVATE (vagg, GST_TYPE_VIDEO_AGGREGATOR, - GstVideoAggregatorPrivate); + vagg->priv = gst_video_aggregator_get_instance_private (vagg); vagg->priv->current_caps = NULL; From b1bd804cb35685b29d33b4b8db71f69defed09f7 Mon Sep 17 00:00:00 2001 From: Daniel Klamt Date: Wed, 25 Apr 2018 16:39:34 +0200 Subject: [PATCH 360/381] Moved the pad offset and aspect ratio to a matrix so it will be added in view space and not in world space https://bugzilla.gnome.org/show_bug.cgi?id=794401 --- ext/gl/gstglvideomixer.c | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/ext/gl/gstglvideomixer.c b/ext/gl/gstglvideomixer.c index 3879dbb079..6cb5c743bb 100644 --- a/ext/gl/gstglvideomixer.c +++ b/ext/gl/gstglvideomixer.c @@ -570,6 +570,7 @@ struct _GstGLVideoMixerPad gboolean geometry_change; GLuint vertex_buffer; + gfloat m_matrix[16]; }; struct _GstGLVideoMixerPadClass @@ -616,6 +617,11 @@ gst_gl_video_mixer_pad_init (GstGLVideoMixerPad * pad) pad->blend_function_src_alpha = DEFAULT_PAD_BLEND_FUNCTION_SRC_ALPHA; pad->blend_function_dst_rgb = DEFAULT_PAD_BLEND_FUNCTION_DST_RGB; pad->blend_function_dst_alpha = DEFAULT_PAD_BLEND_FUNCTION_DST_ALPHA; + memset(pad->m_matrix, 0, sizeof(gfloat) * 4 * 4); + pad->m_matrix[0] = 1.0; + pad->m_matrix[5] = 1.0; + pad->m_matrix[10] = 1.0; + pad->m_matrix[15] = 1.0; } static void @@ -1547,16 +1553,11 @@ gst_gl_video_mixer_callback (gpointer stuff) w = ((gfloat) pad_width / (gfloat) out_width); h = ((gfloat) pad_height / (gfloat) out_height); - /* top-left */ - v_vertices[0] = v_vertices[15] = - 2.0f * (gfloat) pad->xpos / (gfloat) out_width - 1.0f; - /* bottom-left */ - v_vertices[1] = v_vertices[6] = - 2.0f * (gfloat) pad->ypos / (gfloat) out_height - 1.0f; - /* top-right */ - v_vertices[5] = v_vertices[10] = v_vertices[0] + 2.0f * w; - /* bottom-right */ - v_vertices[11] = v_vertices[16] = v_vertices[1] + 2.0f * h; + pad->m_matrix[0] = w; + pad->m_matrix[5] = h; + pad->m_matrix[12] = (gfloat) pad->xpos / (gfloat) out_width; + pad->m_matrix[13] = (gfloat) pad->ypos / (gfloat) out_height; + GST_TRACE ("processing texture:%u dimensions:%ux%u, at %f,%f %fx%f with " "alpha:%f", in_tex, in_width, in_height, v_vertices[0], v_vertices[1], v_vertices[5], v_vertices[11], pad->alpha); @@ -1565,7 +1566,6 @@ gst_gl_video_mixer_callback (gpointer stuff) gl->GenBuffers (1, &pad->vertex_buffer); gl->BindBuffer (GL_ARRAY_BUFFER, pad->vertex_buffer); - gl->BufferData (GL_ARRAY_BUFFER, 4 * 5 * sizeof (GLfloat), v_vertices, GL_STATIC_DRAW); @@ -1583,11 +1583,13 @@ gst_gl_video_mixer_callback (gpointer stuff) { GstVideoAffineTransformationMeta *af_meta; gfloat matrix[16]; + gfloat af_matrix[16]; GstBuffer *buffer = gst_video_aggregator_pad_get_current_buffer (vagg_pad); af_meta = gst_buffer_get_video_affine_transformation_meta (buffer); - gst_gl_get_affine_transformation_meta_as_ndc_ext (af_meta, matrix); + gst_gl_get_affine_transformation_meta_as_ndc_ext (af_meta, af_matrix); + gst_gl_multiply_matrix4(af_matrix, pad->m_matrix, matrix); gst_gl_shader_set_uniform_matrix_4fv (video_mixer->shader, "u_transformation", 1, FALSE, matrix); } From 5958b0bd320c77c52ab9904f99a7865bb74caa27 Mon Sep 17 00:00:00 2001 From: Nicolas Dufresne Date: Tue, 10 Jul 2018 09:42:47 -0400 Subject: [PATCH 361/381] glvideomixer: Add missing string.h include --- ext/gl/gstglvideomixer.c | 1 + 1 file changed, 1 insertion(+) diff --git a/ext/gl/gstglvideomixer.c b/ext/gl/gstglvideomixer.c index 6cb5c743bb..6a068b17c2 100644 --- a/ext/gl/gstglvideomixer.c +++ b/ext/gl/gstglvideomixer.c @@ -43,6 +43,7 @@ #include "config.h" #endif +#include #include #include #include From 443b321ca1f513a640fb02acad18a6a846dbc720 Mon Sep 17 00:00:00 2001 From: Nicolas Dufresne Date: Tue, 10 Jul 2018 09:45:31 -0400 Subject: [PATCH 362/381] glvideomixer: Fix coding style --- ext/gl/gstglvideomixer.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ext/gl/gstglvideomixer.c b/ext/gl/gstglvideomixer.c index 6a068b17c2..68f74edab9 100644 --- a/ext/gl/gstglvideomixer.c +++ b/ext/gl/gstglvideomixer.c @@ -618,7 +618,7 @@ gst_gl_video_mixer_pad_init (GstGLVideoMixerPad * pad) pad->blend_function_src_alpha = DEFAULT_PAD_BLEND_FUNCTION_SRC_ALPHA; pad->blend_function_dst_rgb = DEFAULT_PAD_BLEND_FUNCTION_DST_RGB; pad->blend_function_dst_alpha = DEFAULT_PAD_BLEND_FUNCTION_DST_ALPHA; - memset(pad->m_matrix, 0, sizeof(gfloat) * 4 * 4); + memset (pad->m_matrix, 0, sizeof (gfloat) * 4 * 4); pad->m_matrix[0] = 1.0; pad->m_matrix[5] = 1.0; pad->m_matrix[10] = 1.0; @@ -1558,7 +1558,7 @@ gst_gl_video_mixer_callback (gpointer stuff) pad->m_matrix[5] = h; pad->m_matrix[12] = (gfloat) pad->xpos / (gfloat) out_width; pad->m_matrix[13] = (gfloat) pad->ypos / (gfloat) out_height; - + GST_TRACE ("processing texture:%u dimensions:%ux%u, at %f,%f %fx%f with " "alpha:%f", in_tex, in_width, in_height, v_vertices[0], v_vertices[1], v_vertices[5], v_vertices[11], pad->alpha); @@ -1590,7 +1590,7 @@ gst_gl_video_mixer_callback (gpointer stuff) af_meta = gst_buffer_get_video_affine_transformation_meta (buffer); gst_gl_get_affine_transformation_meta_as_ndc_ext (af_meta, af_matrix); - gst_gl_multiply_matrix4(af_matrix, pad->m_matrix, matrix); + gst_gl_multiply_matrix4 (af_matrix, pad->m_matrix, matrix); gst_gl_shader_set_uniform_matrix_4fv (video_mixer->shader, "u_transformation", 1, FALSE, matrix); } From 32c7249c5e90529a38fb82f1ee027e452b257cba Mon Sep 17 00:00:00 2001 From: Daniel Klamt Date: Wed, 25 Apr 2018 16:36:21 +0200 Subject: [PATCH 363/381] glvideomixer: Moves the objects to zero on z axis Matches the output from a similar glimagesink pipeline when rotating from an upstream gltransformation passed through the affine transformation meta with xpos/ypos being set. https://bugzilla.gnome.org/show_bug.cgi?id=794401 --- ext/gl/gstglvideomixer.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/ext/gl/gstglvideomixer.c b/ext/gl/gstglvideomixer.c index 68f74edab9..fe7915ce7b 100644 --- a/ext/gl/gstglvideomixer.c +++ b/ext/gl/gstglvideomixer.c @@ -1272,10 +1272,10 @@ _draw_checker_background (GstGLVideoMixer * video_mixer) /* *INDENT-OFF* */ gfloat v_vertices[] = { - -1.0,-1.0,-1.0f, - 1.0,-1.0,-1.0f, - 1.0, 1.0,-1.0f, - -1.0, 1.0,-1.0f, + -1.0,-1.0, 0.0f, + 1.0,-1.0, 0.0f, + 1.0, 1.0, 0.0f, + -1.0, 1.0, 0.0f, }; /* *INDENT-ON* */ @@ -1513,10 +1513,10 @@ gst_gl_video_mixer_callback (gpointer stuff) /* *INDENT-OFF* */ gfloat v_vertices[] = { - -1.0,-1.0,-1.0f, 0.0f, 0.0f, - 1.0,-1.0,-1.0f, 1.0f, 0.0f, - 1.0, 1.0,-1.0f, 1.0f, 1.0f, - -1.0, 1.0,-1.0f, 0.0f, 1.0f, + -1.0,-1.0, 0.0f, 0.0f, 0.0f, + 1.0,-1.0, 0.0f, 1.0f, 0.0f, + 1.0, 1.0, 0.0f, 1.0f, 1.0f, + -1.0, 1.0, 0.0f, 0.0f, 1.0f, }; /* *INDENT-ON* */ From 3a3577f826854c0004d374d623aa193a04a18986 Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Thu, 12 Jul 2018 12:48:39 +1000 Subject: [PATCH 364/381] glvideomixer: fix default placement when different sized output i.e. when expanding from 320x240 to 800x600, the resulting frame should appear in the top left corner, not the middle. https://bugzilla.gnome.org/show_bug.cgi?id=794401 --- ext/gl/gstglvideomixer.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/ext/gl/gstglvideomixer.c b/ext/gl/gstglvideomixer.c index fe7915ce7b..ae2397117c 100644 --- a/ext/gl/gstglvideomixer.c +++ b/ext/gl/gstglvideomixer.c @@ -461,8 +461,7 @@ static void gst_gl_video_mixer_child_proxy_init (gpointer g_iface, #define gst_gl_video_mixer_parent_class parent_class G_DEFINE_TYPE_WITH_CODE (GstGLVideoMixer, gst_gl_video_mixer, GST_TYPE_GL_MIXER, G_IMPLEMENT_INTERFACE (GST_TYPE_CHILD_PROXY, - gst_gl_video_mixer_child_proxy_init); - DEBUG_INIT); + gst_gl_video_mixer_child_proxy_init); DEBUG_INIT); static void gst_gl_video_mixer_set_property (GObject * object, guint prop_id, const GValue * value, GParamSpec * pspec); @@ -1556,12 +1555,14 @@ gst_gl_video_mixer_callback (gpointer stuff) pad->m_matrix[0] = w; pad->m_matrix[5] = h; - pad->m_matrix[12] = (gfloat) pad->xpos / (gfloat) out_width; - pad->m_matrix[13] = (gfloat) pad->ypos / (gfloat) out_height; + pad->m_matrix[12] = + 2. * (gfloat) pad->xpos / (gfloat) out_width - (1. - w); + pad->m_matrix[13] = + 2. * (gfloat) pad->ypos / (gfloat) out_height - (1. - h); GST_TRACE ("processing texture:%u dimensions:%ux%u, at %f,%f %fx%f with " - "alpha:%f", in_tex, in_width, in_height, v_vertices[0], v_vertices[1], - v_vertices[5], v_vertices[11], pad->alpha); + "alpha:%f", in_tex, in_width, in_height, pad->m_matrix[12], + pad->m_matrix[13], pad->m_matrix[0], pad->m_matrix[5], pad->alpha); if (!pad->vertex_buffer) gl->GenBuffers (1, &pad->vertex_buffer); From 4380d5777980b7dd29e037a40d8dffa33dc89bec Mon Sep 17 00:00:00 2001 From: Seungha Yang Date: Wed, 18 Jul 2018 20:05:26 +0900 Subject: [PATCH 365/381] compositor: Update conversion info in property setter ... not in getter. Otherwise, video-converter will not be updated with new width/height https://bugzilla.gnome.org/show_bug.cgi?id=796828 --- gst/compositor/compositor.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/gst/compositor/compositor.c b/gst/compositor/compositor.c index 341dc70fa3..105fc40571 100644 --- a/gst/compositor/compositor.c +++ b/gst/compositor/compositor.c @@ -158,13 +158,9 @@ gst_compositor_pad_get_property (GObject * object, guint prop_id, break; case PROP_PAD_WIDTH: g_value_set_int (value, pad->width); - gst_video_aggregator_convert_pad_update_conversion_info - (GST_VIDEO_AGGREGATOR_CONVERT_PAD (pad)); break; case PROP_PAD_HEIGHT: g_value_set_int (value, pad->height); - gst_video_aggregator_convert_pad_update_conversion_info - (GST_VIDEO_AGGREGATOR_CONVERT_PAD (pad)); break; case PROP_PAD_ALPHA: g_value_set_double (value, pad->alpha); @@ -193,9 +189,13 @@ gst_compositor_pad_set_property (GObject * object, guint prop_id, break; case PROP_PAD_WIDTH: pad->width = g_value_get_int (value); + gst_video_aggregator_convert_pad_update_conversion_info + (GST_VIDEO_AGGREGATOR_CONVERT_PAD (pad)); break; case PROP_PAD_HEIGHT: pad->height = g_value_get_int (value); + gst_video_aggregator_convert_pad_update_conversion_info + (GST_VIDEO_AGGREGATOR_CONVERT_PAD (pad)); break; case PROP_PAD_ALPHA: pad->alpha = g_value_get_double (value); From aa3e64c71de7b072f421866c3c9c34b8969c1bd0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Fri, 20 Jul 2018 16:25:02 +0300 Subject: [PATCH 366/381] compositor: Use 255 as maximum alpha instead of 256 255 will easily become 0 in the blending function as they expect the maximum value to be 255. Can be reproduce with gst-launch-1.0 videotestsrc pattern=ball ! c.sink_0 \ videotestsrc pattern=snow ! c.sink_1 \ compositor name=c \ sink_0::zorder=0 sink_1::zorder=1 sink_0::crossfade-ratio=0.5 \ background=black ! \ videoconvert ! xvimagesink crossfade-ratio +/- 0.001 makes it work correctly and the same happens at e.g. 0.25, 0.75, N*0.0625 https://bugzilla.gnome.org/show_bug.cgi?id=796846 --- gst/compositor/blend.c | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/gst/compositor/blend.c b/gst/compositor/blend.c index b36c164654..416f78cfb1 100644 --- a/gst/compositor/blend.c +++ b/gst/compositor/blend.c @@ -33,8 +33,6 @@ #include -#define BLEND(D,S,alpha) (((D) * (256 - (alpha)) + (S) * (alpha)) >> 8) - GST_DEBUG_CATEGORY_STATIC (gst_compositor_blend_debug); #define GST_CAT_DEFAULT gst_compositor_blend_debug @@ -61,7 +59,7 @@ method##_ ##name (GstVideoFrame * srcframe, gint xpos, gint ypos, \ dest_width = GST_VIDEO_FRAME_COMP_WIDTH (destframe, 0); \ dest_height = GST_VIDEO_FRAME_COMP_HEIGHT (destframe, 0); \ \ - s_alpha = CLAMP ((gint) (src_alpha * 256), 0, 256); \ + s_alpha = CLAMP ((gint) (src_alpha * 255), 0, 255); \ \ /* If it's completely transparent... we just return */ \ if (G_UNLIKELY (s_alpha == 0)) \ @@ -247,7 +245,7 @@ _blend_##format_name (const guint8 * src, guint8 * dest, \ return; \ } \ \ - b_alpha = CLAMP ((gint) (src_alpha * 256), 0, 256); \ + b_alpha = CLAMP ((gint) (src_alpha * 255), 0, 255); \ \ BLENDLOOP(dest, dest_stride, src, src_stride, b_alpha, src_width, src_height);\ } \ @@ -492,7 +490,7 @@ _blend_##format_name (const guint8 * src, guint8 * dest, \ return; \ } \ \ - b_alpha = CLAMP ((gint) (src_alpha * 256), 0, 256); \ + b_alpha = CLAMP ((gint) (src_alpha * 255), 0, 255); \ \ BLENDLOOP(dest, dest_stride, src, src_stride, b_alpha, src_width, src_height); \ } \ @@ -691,7 +689,7 @@ blend_##name (GstVideoFrame * srcframe, gint xpos, gint ypos, \ src_stride = GST_VIDEO_FRAME_COMP_STRIDE (srcframe, 0); \ dest_stride = GST_VIDEO_FRAME_COMP_STRIDE (destframe, 0); \ \ - b_alpha = CLAMP ((gint) (src_alpha * 256), 0, 256); \ + b_alpha = CLAMP ((gint) (src_alpha * 255), 0, 255); \ \ /* adjust src pointers for negative sizes */ \ if (xpos < 0) { \ @@ -857,7 +855,7 @@ blend_##name (GstVideoFrame * srcframe, gint xpos, gint ypos, \ src_stride = GST_VIDEO_FRAME_COMP_STRIDE (srcframe, 0); \ dest_stride = GST_VIDEO_FRAME_COMP_STRIDE (destframe, 0); \ \ - b_alpha = CLAMP ((gint) (src_alpha * 256), 0, 256); \ + b_alpha = CLAMP ((gint) (src_alpha * 255), 0, 255); \ \ xpos = GST_ROUND_UP_2 (xpos); \ \ From 3e067ae333117ca58826ac75553784fa165111dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Thu, 26 Jul 2018 00:20:02 +0300 Subject: [PATCH 367/381] compositor: Don't leak all buffers while crossfading and not all pads are crossfading --- gst/compositor/compositor.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/gst/compositor/compositor.c b/gst/compositor/compositor.c index 105fc40571..ec62de51ba 100644 --- a/gst/compositor/compositor.c +++ b/gst/compositor/compositor.c @@ -833,8 +833,13 @@ gst_compositor_fill_transparent (GstCompositor * self, GstVideoFrame * frame, if (!gst_video_frame_map (nframe, &frame->info, cbuffer, GST_MAP_WRITE)) { GST_WARNING_OBJECT (self, "Could not map output buffer"); + gst_buffer_unref (cbuffer); return GST_FLOW_ERROR; } + + /* the last reference is owned by the frame and released once the frame + * is unmapped. We leak it if we don't unref here */ + gst_buffer_unref (cbuffer); } else { nframe = frame; } From 8430e470063984560de61620fe2d72c03f3901da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Thu, 16 Aug 2018 10:02:42 +0300 Subject: [PATCH 368/381] videoaggregator: Make sure to hold object lock while iterating sink pads They might otherwise just change while we iterate. --- gst-libs/gst/video/gstvideoaggregator.c | 28 +++++++++++++++++-------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c index dafd59d933..2ade7da9f4 100644 --- a/gst-libs/gst/video/gstvideoaggregator.c +++ b/gst-libs/gst/video/gstvideoaggregator.c @@ -949,6 +949,20 @@ gst_video_aggregator_default_update_src_caps (GstAggregator * agg, return GST_FLOW_OK; } +static gboolean +_update_conversion_info (GstElement * element, GstPad * pad, gpointer user_data) +{ + GstVideoAggregatorPad *vaggpad = GST_VIDEO_AGGREGATOR_PAD (pad); + GstVideoAggregatorPadClass *vaggpad_klass = + GST_VIDEO_AGGREGATOR_PAD_GET_CLASS (vaggpad); + + if (vaggpad_klass->update_conversion_info) { + vaggpad_klass->update_conversion_info (vaggpad); + } + + return TRUE; +} + static gboolean gst_video_aggregator_default_negotiated_src_caps (GstAggregator * agg, GstCaps * caps) @@ -1001,15 +1015,8 @@ gst_video_aggregator_default_negotiated_src_caps (GstAggregator * agg, } /* Then browse the sinks once more, setting or unsetting conversion if needed */ - for (l = GST_ELEMENT (vagg)->sinkpads; l; l = l->next) { - GstVideoAggregatorPad *pad = GST_VIDEO_AGGREGATOR_PAD (l->data); - GstVideoAggregatorPadClass *vaggpad_klass = - GST_VIDEO_AGGREGATOR_PAD_GET_CLASS (pad); - - if (vaggpad_klass->update_conversion_info) { - vaggpad_klass->update_conversion_info (pad); - } - } + gst_element_foreach_sink_pad (GST_ELEMENT_CAST (vagg), + _update_conversion_info, NULL); if (vagg->priv->current_caps == NULL || gst_caps_is_equal (caps, vagg->priv->current_caps) == FALSE) { @@ -1032,6 +1039,7 @@ gst_video_aggregator_get_sinkpads_interlace_mode (GstVideoAggregator * vagg, { GList *walk; + GST_OBJECT_LOCK (vagg); for (walk = GST_ELEMENT (vagg)->sinkpads; walk; walk = g_list_next (walk)) { GstVideoAggregatorPad *vaggpad = walk->data; @@ -1040,9 +1048,11 @@ gst_video_aggregator_get_sinkpads_interlace_mode (GstVideoAggregator * vagg, if (vaggpad->info.finfo && GST_VIDEO_INFO_FORMAT (&vaggpad->info) != GST_VIDEO_FORMAT_UNKNOWN) { *mode = GST_VIDEO_INFO_INTERLACE_MODE (&vaggpad->info); + GST_OBJECT_UNLOCK (vagg); return TRUE; } } + GST_OBJECT_UNLOCK (vagg); return FALSE; } From 5485a2b9eefe4229a0ea0bd5959544a34d757403 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Thu, 16 Aug 2018 17:07:06 +0300 Subject: [PATCH 369/381] compositor: Define crossfade-ratio to have range [0.0,1.0] Previously negative values had the same effect as 0.0, which was confusing. https://bugzilla.gnome.org/show_bug.cgi?id=796845 --- gst/compositor/compositor.c | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/gst/compositor/compositor.c b/gst/compositor/compositor.c index ec62de51ba..c831738ae4 100644 --- a/gst/compositor/compositor.c +++ b/gst/compositor/compositor.c @@ -128,7 +128,7 @@ static void gst_compositor_child_proxy_init (gpointer g_iface, #define DEFAULT_PAD_WIDTH 0 #define DEFAULT_PAD_HEIGHT 0 #define DEFAULT_PAD_ALPHA 1.0 -#define DEFAULT_PAD_CROSSFADE_RATIO -1.0 +#define DEFAULT_PAD_CROSSFADE_RATIO 0.0 enum { PROP_PAD_0, @@ -203,7 +203,7 @@ gst_compositor_pad_set_property (GObject * object, guint prop_id, case PROP_PAD_CROSSFADE_RATIO: pad->crossfade = g_value_get_double (value); gst_video_aggregator_pad_set_needs_alpha (GST_VIDEO_AGGREGATOR_PAD (pad), - pad->crossfade >= 0.0f); + pad->crossfade > 0.0); break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); @@ -336,8 +336,8 @@ gst_compositor_pad_prepare_frame (GstVideoAggregatorPad * pad, GST_OBJECT_LOCK (vagg); /* Check if we are crossfading the pad one way or another */ l = g_list_find (GST_ELEMENT (vagg)->sinkpads, pad); - if ((l->prev && GST_COMPOSITOR_PAD (l->prev->data)->crossfade >= 0.0) || - (GST_COMPOSITOR_PAD (pad)->crossfade >= 0.0)) { + if ((l->prev && GST_COMPOSITOR_PAD (l->prev->data)->crossfade > 0.0) || + (GST_COMPOSITOR_PAD (pad)->crossfade > 0.0)) { GST_DEBUG_OBJECT (pad, "Is being crossfaded with previous pad"); l = NULL; } else { @@ -471,9 +471,8 @@ gst_compositor_pad_class_init (GstCompositorPadClass * klass) G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); g_object_class_install_property (gobject_class, PROP_PAD_CROSSFADE_RATIO, g_param_spec_double ("crossfade-ratio", "Crossfade ratio", - "The crossfade ratio to use while crossfading with the following pad." - "A value inferior to 0 means no crossfading.", - -1.0, 1.0, DEFAULT_PAD_CROSSFADE_RATIO, + "The crossfade ratio to use while crossfading with the following pad", + 0.0, 1.0, DEFAULT_PAD_CROSSFADE_RATIO, G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); vaggpadclass->prepare_frame = @@ -881,8 +880,8 @@ gst_compositor_crossfade_frames (GstCompositor * self, GstVideoFrame * outframe) for (l = GST_ELEMENT (self)->sinkpads; l; l = l->next) { GstCompositorPad *compo_pad = GST_COMPOSITOR_PAD (l->data); - if (compo_pad->crossfade < 0.0 && l->next && - GST_COMPOSITOR_PAD (l->next->data)->crossfade < 0) { + if (compo_pad->crossfade == 0.0 && l->next && + GST_COMPOSITOR_PAD (l->next->data)->crossfade == 0.0) { all_crossfading = FALSE; break; @@ -898,7 +897,7 @@ gst_compositor_crossfade_frames (GstCompositor * self, GstVideoFrame * outframe) GstVideoFrame *prepared_frame = gst_video_aggregator_pad_get_prepared_frame (pad); - if (compo_pad->crossfade >= 0.0f && prepared_frame) { + if (compo_pad->crossfade > 0.0 && prepared_frame) { gfloat alpha = compo_pad->crossfade * compo_pad->alpha; GstVideoAggregatorPad *npad = l->next ? l->next->data : NULL; GstVideoFrame *next_prepared_frame; From 05d0c72a597214f6f072eab4b92c0796477ca540 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Wed, 19 Sep 2018 10:28:56 +0300 Subject: [PATCH 370/381] glvideomixer: Fix typo in property description --- ext/gl/gstglvideomixer.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ext/gl/gstglvideomixer.c b/ext/gl/gstglvideomixer.c index ae2397117c..cf634d8e42 100644 --- a/ext/gl/gstglvideomixer.c +++ b/ext/gl/gstglvideomixer.c @@ -289,7 +289,7 @@ gst_gl_video_mixer_input_class_init (GstGLVideoMixerInputClass * klass) PROP_INPUT_BLEND_FUNCTION_DST_ALPHA, g_param_spec_enum ("blend-function-dst-alpha", "Blend Function Destination Alpha", - "Blend Function for Destiniation Alpha", + "Blend Function for Destination Alpha", GST_TYPE_GL_VIDEO_MIXER_BLEND_FUNCTION, DEFAULT_PAD_BLEND_FUNCTION_DST_ALPHA, G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); @@ -690,7 +690,7 @@ gst_gl_video_mixer_pad_class_init (GstGLVideoMixerPadClass * klass) PROP_INPUT_BLEND_FUNCTION_DST_ALPHA, g_param_spec_enum ("blend-function-dst-alpha", "Blend Function Destination Alpha", - "Blend Function for Destiniation Alpha", + "Blend Function for Destination Alpha", GST_TYPE_GL_VIDEO_MIXER_BLEND_FUNCTION, DEFAULT_PAD_BLEND_FUNCTION_DST_ALPHA, G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); From f757da11602f4c110eb79d9644a8ad70e1035b53 Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Wed, 19 Sep 2018 18:01:26 +1000 Subject: [PATCH 371/381] glvideomixer: fix constant alpha enum value for constant alpha --- ext/gl/gstglvideomixer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/gl/gstglvideomixer.c b/ext/gl/gstglvideomixer.c index cf634d8e42..2256e549b5 100644 --- a/ext/gl/gstglvideomixer.c +++ b/ext/gl/gstglvideomixer.c @@ -137,7 +137,7 @@ gst_gl_video_mixer_blend_function_get_type (void) "One Minus Constant Color", "one-minus-contant-color"}, {GST_GL_VIDEO_MIXER_BLEND_FUNCTION_CONSTANT_ALPHA, "Constant Alpha", "constant-alpha"}, - {GST_GL_VIDEO_MIXER_BLEND_FUNCTION_ONE_MINUS_CONSTANT_COLOR, + {GST_GL_VIDEO_MIXER_BLEND_FUNCTION_ONE_MINUS_CONSTANT_ALPHA, "One Minus Constant Alpha", "one-minus-contant-alpha"}, {GST_GL_VIDEO_MIXER_BLEND_FUNCTION_SRC_ALPHA_SATURATE, "Source Alpha Saturate", "src-alpha-saturate"}, From b59fcdc789190f83632dfd282994086f845b2143 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Mon, 24 Sep 2018 23:26:10 +0300 Subject: [PATCH 372/381] glmixerbin: "latency" property on aggregator is uint64, not int64 Also update the property description and range with aggregator's values. --- ext/gl/gstglmixerbin.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/ext/gl/gstglmixerbin.c b/ext/gl/gstglmixerbin.c index 94525e1de0..866b4b3f0f 100644 --- a/ext/gl/gstglmixerbin.c +++ b/ext/gl/gstglmixerbin.c @@ -178,11 +178,10 @@ gst_gl_mixer_bin_class_init (GstGLMixerBinClass * klass) G_PARAM_STATIC_STRINGS)); g_object_class_install_property (gobject_class, PROP_LATENCY, - g_param_spec_int64 ("latency", "Buffer latency", + g_param_spec_uint64 ("latency", "Buffer latency", "Additional latency in live mode to allow upstream " "to take longer to produce buffers for the current " - "position", 0, - (G_MAXLONG == G_MAXINT64) ? G_MAXINT64 : (G_MAXLONG * GST_SECOND - 1), + "position (in nanoseconds)", 0, G_MAXUINT64, DEFAULT_LATENCY, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); g_object_class_install_property (gobject_class, PROP_START_TIME_SELECTION, From b37c08ecb5d62b2f34dad8865c6adf1e354937f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Wed, 3 Oct 2018 16:00:23 +0300 Subject: [PATCH 373/381] glstereomix: Fix build after deprecating misnamed GST_TYPE_GL_STEREO_DOWNMIX_MODE_TYPE --- ext/gl/gstglstereomix.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/gl/gstglstereomix.c b/ext/gl/gstglstereomix.c index e2bf105d95..464fad1354 100644 --- a/ext/gl/gstglstereomix.c +++ b/ext/gl/gstglstereomix.c @@ -185,7 +185,7 @@ gst_gl_stereo_mix_class_init (GstGLStereoMixClass * klass) g_object_class_install_property (gobject_class, PROP_DOWNMIX_MODE, g_param_spec_enum ("downmix-mode", "Mode for mono downmixed output", "Output anaglyph type to generate when downmixing to mono", - GST_TYPE_GL_STEREO_DOWNMIX_MODE_TYPE, DEFAULT_DOWNMIX, + GST_TYPE_GL_STEREO_DOWNMIX, DEFAULT_DOWNMIX, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); gst_element_class_add_static_pad_template_with_gtype (element_class, From 2f88e2ae16818f0ace3643fc1e2b5b500b7a5dd5 Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Thu, 27 Sep 2018 16:37:28 +1000 Subject: [PATCH 374/381] glmixerbin: add gloverlaycompositor to each input stream Flattens the overlay compositions into the stream before the mixer will mix them. https://bugzilla.gnome.org/show_bug.cgi?id=759867 --- ext/gl/gstglmixerbin.c | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/ext/gl/gstglmixerbin.c b/ext/gl/gstglmixerbin.c index 866b4b3f0f..47a390d2a0 100644 --- a/ext/gl/gstglmixerbin.c +++ b/ext/gl/gstglmixerbin.c @@ -68,6 +68,7 @@ struct input_chain GstGhostPad *ghost_pad; GstElement *upload; GstElement *in_convert; + GstElement *in_overlay; GstPad *mixer_pad; }; @@ -91,6 +92,12 @@ _free_input_chain (struct input_chain *chain) chain->in_convert = NULL; } + if (chain->in_overlay) { + gst_element_set_state (chain->in_overlay, GST_STATE_NULL); + gst_bin_remove (GST_BIN (chain->self), chain->in_overlay); + chain->in_overlay = NULL; + } + if (chain->mixer_pad) { gst_element_release_request_pad (chain->self->mixer, chain->mixer_pad); gst_object_unref (chain->mixer_pad); @@ -291,16 +298,21 @@ _create_input_chain (GstGLMixerBin * self, struct input_chain *chain, chain->upload = gst_element_factory_make ("glupload", NULL); chain->in_convert = gst_element_factory_make ("glcolorconvert", NULL); + chain->in_overlay = gst_element_factory_make ("gloverlaycompositor", NULL); res &= gst_bin_add (GST_BIN (self), chain->in_convert); + res &= gst_bin_add (GST_BIN (self), chain->in_overlay); res &= gst_bin_add (GST_BIN (self), chain->upload); - pad = gst_element_get_static_pad (chain->in_convert, "src"); + pad = gst_element_get_static_pad (chain->in_overlay, "src"); if (gst_pad_link (pad, mixer_pad) != GST_PAD_LINK_OK) { gst_object_unref (pad); return FALSE; } gst_object_unref (pad); + res &= + gst_element_link_pads (chain->in_convert, "src", chain->in_overlay, + "sink"); res &= gst_element_link_pads (chain->upload, "src", chain->in_convert, "sink"); @@ -332,6 +344,7 @@ _create_input_chain (GstGLMixerBin * self, struct input_chain *chain, gst_element_sync_state_with_parent (chain->upload); gst_element_sync_state_with_parent (chain->in_convert); + gst_element_sync_state_with_parent (chain->in_overlay); return TRUE; } From 89c14c955bab80c2e14f6d76b07dadf23ba985bc Mon Sep 17 00:00:00 2001 From: Matthew Waters Date: Thu, 4 Oct 2018 23:45:49 +1000 Subject: [PATCH 375/381] glvideomixer: fix the default blend modes for unpremultipled alpha Alpha in GStreamer is unpremultiplied --- ext/gl/gstglvideomixer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/gl/gstglvideomixer.c b/ext/gl/gstglvideomixer.c index 2256e549b5..0306d22e53 100644 --- a/ext/gl/gstglvideomixer.c +++ b/ext/gl/gstglvideomixer.c @@ -162,7 +162,7 @@ gst_gl_video_mixer_blend_function_get_type (void) #define DEFAULT_PAD_BLEND_EQUATION_RGB GST_GL_VIDEO_MIXER_BLEND_EQUATION_ADD #define DEFAULT_PAD_BLEND_EQUATION_ALPHA GST_GL_VIDEO_MIXER_BLEND_EQUATION_ADD #define DEFAULT_PAD_BLEND_FUNCTION_SRC_RGB GST_GL_VIDEO_MIXER_BLEND_FUNCTION_SRC_ALPHA -#define DEFAULT_PAD_BLEND_FUNCTION_SRC_ALPHA GST_GL_VIDEO_MIXER_BLEND_FUNCTION_SRC_ALPHA +#define DEFAULT_PAD_BLEND_FUNCTION_SRC_ALPHA GST_GL_VIDEO_MIXER_BLEND_FUNCTION_ONE #define DEFAULT_PAD_BLEND_FUNCTION_DST_RGB GST_GL_VIDEO_MIXER_BLEND_FUNCTION_ONE_MINUS_SRC_ALPHA #define DEFAULT_PAD_BLEND_FUNCTION_DST_ALPHA GST_GL_VIDEO_MIXER_BLEND_FUNCTION_ONE_MINUS_SRC_ALPHA From 5adec0db4f069e8f624c62ca0b4a6d4345542f75 Mon Sep 17 00:00:00 2001 From: Seungha Yang Date: Wed, 17 Oct 2018 21:09:22 +0900 Subject: [PATCH 376/381] meson: Replace empty configuration_data() with copy keyword Use 'copy' keyword to avoid meson warning message. Note that 'copy' keyword in configure_file() is available since meson 0.47.0 https://bugzilla.gnome.org/show_bug.cgi?id=797298 --- gst/compositor/meson.build | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gst/compositor/meson.build b/gst/compositor/meson.build index 2af1b68e32..f379a31a07 100644 --- a/gst/compositor/meson.build +++ b/gst/compositor/meson.build @@ -16,10 +16,10 @@ if have_orcc else orc_h = configure_file(input : orcsrc + '-dist.h', output : orcsrc + '.h', - configuration : configuration_data()) + copy : true) orc_c = configure_file(input : orcsrc + '-dist.c', output : orcsrc + '.c', - configuration : configuration_data()) + copy : true) endif gstcompositor = library('gstcompositor', From c41439d16f8fada1db6bc498e645bc518c1a88aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Sat, 27 Oct 2018 12:37:07 +0100 Subject: [PATCH 377/381] compositor: Remove extra header for the pad declaration There's no reason for having this separate apart from making things less discoverable. --- gst/compositor/Makefile.am | 3 +- gst/compositor/compositor.c | 1 - gst/compositor/compositor.h | 37 ++++++++++++++++++ gst/compositor/compositorpad.h | 68 ---------------------------------- 4 files changed, 38 insertions(+), 71 deletions(-) delete mode 100644 gst/compositor/compositorpad.h diff --git a/gst/compositor/Makefile.am b/gst/compositor/Makefile.am index 6b98b78bbc..193d153d95 100644 --- a/gst/compositor/Makefile.am +++ b/gst/compositor/Makefile.am @@ -25,5 +25,4 @@ libgstcompositor_la_LDFLAGS = $(GST_PLUGIN_LDFLAGS) # headers we need but don't want installed noinst_HEADERS = \ blend.h \ - compositor.h \ - compositorpad.h + compositor.h diff --git a/gst/compositor/compositor.c b/gst/compositor/compositor.c index c831738ae4..d036d68437 100644 --- a/gst/compositor/compositor.c +++ b/gst/compositor/compositor.c @@ -93,7 +93,6 @@ #include #include "compositor.h" -#include "compositorpad.h" #ifdef DISABLE_ORC #define orc_memset memset diff --git a/gst/compositor/compositor.h b/gst/compositor/compositor.h index d8cefa3d2a..37ffcb5c4c 100644 --- a/gst/compositor/compositor.h +++ b/gst/compositor/compositor.h @@ -39,9 +39,22 @@ G_BEGIN_DECLS #define GST_IS_COMPOSITOR_CLASS(klass) \ (G_TYPE_CHECK_CLASS_TYPE((klass),GST_TYPE_COMPOSITOR)) +#define GST_TYPE_COMPOSITOR_PAD (gst_compositor_pad_get_type()) +#define GST_COMPOSITOR_PAD(obj) \ + (G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_COMPOSITOR_PAD, GstCompositorPad)) +#define GST_COMPOSITOR_PAD_CLASS(klass) \ + (G_TYPE_CHECK_CLASS_CAST((klass),GST_TYPE_COMPOSITOR_PAD, GstCompositorPadClass)) +#define GST_IS_COMPOSITOR_PAD(obj) \ + (G_TYPE_CHECK_INSTANCE_TYPE((obj),GST_TYPE_COMPOSITOR_PAD)) +#define GST_IS_COMPOSITOR_PAD_CLASS(klass) \ + (G_TYPE_CHECK_CLASS_TYPE((klass),GST_TYPE_COMPOSITOR_PAD)) + typedef struct _GstCompositor GstCompositor; typedef struct _GstCompositorClass GstCompositorClass; +typedef struct _GstCompositorPad GstCompositorPad; +typedef struct _GstCompositorPadClass GstCompositorPadClass; + /** * GstcompositorBackground: * @COMPOSITOR_BACKGROUND_CHECKER: checker pattern background @@ -79,7 +92,31 @@ struct _GstCompositorClass GstVideoAggregatorClass parent_class; }; +/** + * GstCompositorPad: + * + * The opaque #GstCompositorPad structure. + */ +struct _GstCompositorPad +{ + GstVideoAggregatorConvertPad parent; + + /* properties */ + gint xpos, ypos; + gint width, height; + gdouble alpha; + gdouble crossfade; + + gboolean crossfaded; +}; + +struct _GstCompositorPadClass +{ + GstVideoAggregatorConvertPadClass parent_class; +}; + GType gst_compositor_get_type (void); +GType gst_compositor_pad_get_type (void); G_END_DECLS #endif /* __GST_COMPOSITOR_H__ */ diff --git a/gst/compositor/compositorpad.h b/gst/compositor/compositorpad.h deleted file mode 100644 index 88be0e25ab..0000000000 --- a/gst/compositor/compositorpad.h +++ /dev/null @@ -1,68 +0,0 @@ -/* Generic compositor plugin pad - * Copyright (C) 2008 Wim Taymans - * Copyright (C) 2010 Sebastian Dröge - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Library General Public - * License as published by the Free Software Foundation; either - * version 2 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Library General Public License for more details. - * - * You should have received a copy of the GNU Library General Public - * License along with this library; if not, write to the - * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, - * Boston, MA 02110-1301, USA. - */ - -#ifndef __GST_COMPOSITOR_PAD_H__ -#define __GST_COMPOSITOR_PAD_H__ - -#include -#include - -G_BEGIN_DECLS - -#define GST_TYPE_COMPOSITOR_PAD (gst_compositor_pad_get_type()) -#define GST_COMPOSITOR_PAD(obj) \ - (G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_COMPOSITOR_PAD, GstCompositorPad)) -#define GST_COMPOSITOR_PAD_CLASS(klass) \ - (G_TYPE_CHECK_CLASS_CAST((klass),GST_TYPE_COMPOSITOR_PAD, GstCompositorPadClass)) -#define GST_IS_COMPOSITOR_PAD(obj) \ - (G_TYPE_CHECK_INSTANCE_TYPE((obj),GST_TYPE_COMPOSITOR_PAD)) -#define GST_IS_COMPOSITOR_PAD_CLASS(klass) \ - (G_TYPE_CHECK_CLASS_TYPE((klass),GST_TYPE_COMPOSITOR_PAD)) - -typedef struct _GstCompositorPad GstCompositorPad; -typedef struct _GstCompositorPadClass GstCompositorPadClass; - -/** - * GstCompositorPad: - * - * The opaque #GstCompositorPad structure. - */ -struct _GstCompositorPad -{ - GstVideoAggregatorConvertPad parent; - - /* properties */ - gint xpos, ypos; - gint width, height; - gdouble alpha; - gdouble crossfade; - - gboolean crossfaded; -}; - -struct _GstCompositorPadClass -{ - GstVideoAggregatorConvertPadClass parent_class; -}; - -GType gst_compositor_pad_get_type (void); - -G_END_DECLS -#endif /* __GST_COMPOSITOR_PAD_H__ */ From e87fa73e606251036876cccf6d743d6a1eee3b6b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Sat, 27 Oct 2018 16:44:27 +0100 Subject: [PATCH 378/381] compositor: Implement different operators via per-pad property This removes the crossfade-ratio property and replaces it with an operator property. Currently this implements the following operators: - SOURCE: Copy over the source and don't look at the destination - OVER: Default blending of the source over the destination - ADD: Like OVER but simply adding the alpha instead See the example for how to implement crossfading with this. https://bugzilla.gnome.org/show_bug.cgi?id=797169 --- gst/compositor/blend.c | 64 +++++-- gst/compositor/blend.h | 15 +- gst/compositor/compositor.c | 249 ++++++++------------------ gst/compositor/compositor.h | 27 ++- gst/compositor/compositororc.orc | 60 +++++++ tests/examples/compositor/crossfade.c | 38 ++-- 6 files changed, 236 insertions(+), 217 deletions(-) diff --git a/gst/compositor/blend.c b/gst/compositor/blend.c index 416f78cfb1..c54d8988ee 100644 --- a/gst/compositor/blend.c +++ b/gst/compositor/blend.c @@ -100,32 +100,46 @@ _overlay_loop_##name (guint8 * dest, const guint8 * src, gint src_height, \ { \ s_alpha = MIN (255, s_alpha); \ switch (mode) { \ - case COMPOSITOR_BLEND_MODE_NORMAL:\ + case COMPOSITOR_BLEND_MODE_SOURCE:\ + compositor_orc_source_##name (dest, dest_stride, src, src_stride, \ + s_alpha, src_width, src_height); \ + break;\ + case COMPOSITOR_BLEND_MODE_OVER:\ compositor_orc_overlay_##name (dest, dest_stride, src, src_stride, \ s_alpha, src_width, src_height); \ break;\ - case COMPOSITOR_BLEND_MODE_ADDITIVE:\ + case COMPOSITOR_BLEND_MODE_ADD:\ compositor_orc_overlay_##name##_addition (dest, dest_stride, src, src_stride, \ s_alpha, src_width, src_height); \ break;\ }\ } -#define BLEND_A32_LOOP_WITH_MODE(name) \ +#define BLEND_A32_LOOP(name) \ static inline void \ _blend_loop_##name (guint8 * dest, const guint8 * src, gint src_height, \ gint src_width, gint src_stride, gint dest_stride, guint s_alpha, \ GstCompositorBlendMode mode) \ { \ s_alpha = MIN (255, s_alpha); \ - compositor_orc_blend_##name (dest, dest_stride, src, src_stride, \ - s_alpha, src_width, src_height); \ + switch (mode) { \ + case COMPOSITOR_BLEND_MODE_SOURCE:\ + compositor_orc_source_##name (dest, dest_stride, src, src_stride, \ + s_alpha, src_width, src_height); \ + break;\ + case COMPOSITOR_BLEND_MODE_OVER:\ + case COMPOSITOR_BLEND_MODE_ADD:\ + /* both modes are the same for opaque background */ \ + compositor_orc_blend_##name (dest, dest_stride, src, src_stride, \ + s_alpha, src_width, src_height); \ + break;\ + }\ } OVERLAY_A32_LOOP (argb); OVERLAY_A32_LOOP (bgra); -BLEND_A32_LOOP_WITH_MODE (argb); -BLEND_A32_LOOP_WITH_MODE (bgra); +BLEND_A32_LOOP (argb); +BLEND_A32_LOOP (bgra); #if G_BYTE_ORDER == G_LITTLE_ENDIAN BLEND_A32 (argb, blend, _blend_loop_argb); @@ -223,11 +237,16 @@ A32_COLOR (ayuv, FALSE, 24, 16, 8, 0); inline static void \ _blend_##format_name (const guint8 * src, guint8 * dest, \ gint src_stride, gint dest_stride, gint src_width, gint src_height, \ - gdouble src_alpha) \ + gdouble src_alpha, GstCompositorBlendMode mode) \ { \ gint i; \ gint b_alpha; \ \ + /* in source mode we just have to copy over things */ \ + if (mode == COMPOSITOR_BLEND_MODE_SOURCE) { \ + src_alpha = 1.0; \ + } \ + \ /* If it's completely transparent... we just return */ \ if (G_UNLIKELY (src_alpha == 0.0)) { \ GST_INFO ("Fast copy (alpha == 0.0)"); \ @@ -324,7 +343,7 @@ blend_##format_name (GstVideoFrame * srcframe, gint xpos, gint ypos, \ b_dest + comp_xpos + comp_ypos * dest_comp_rowstride, \ src_comp_rowstride, \ dest_comp_rowstride, src_comp_width, src_comp_height, \ - src_alpha); \ + src_alpha, mode); \ \ b_src = GST_VIDEO_FRAME_COMP_DATA (srcframe, 1); \ b_dest = GST_VIDEO_FRAME_COMP_DATA (destframe, 1); \ @@ -340,7 +359,7 @@ blend_##format_name (GstVideoFrame * srcframe, gint xpos, gint ypos, \ b_dest + comp_xpos + comp_ypos * dest_comp_rowstride, \ src_comp_rowstride, \ dest_comp_rowstride, src_comp_width, src_comp_height, \ - src_alpha); \ + src_alpha, mode); \ \ b_src = GST_VIDEO_FRAME_COMP_DATA (srcframe, 2); \ b_dest = GST_VIDEO_FRAME_COMP_DATA (destframe, 2); \ @@ -356,7 +375,7 @@ blend_##format_name (GstVideoFrame * srcframe, gint xpos, gint ypos, \ b_dest + comp_xpos + comp_ypos * dest_comp_rowstride, \ src_comp_rowstride, \ dest_comp_rowstride, src_comp_width, src_comp_height, \ - src_alpha); \ + src_alpha, mode); \ } #define PLANAR_YUV_FILL_CHECKER(format_name, format_enum, MEMSET) \ @@ -468,11 +487,16 @@ PLANAR_YUV_FILL_COLOR (y41b, GST_VIDEO_FORMAT_Y41B, memset); inline static void \ _blend_##format_name (const guint8 * src, guint8 * dest, \ gint src_stride, gint dest_stride, gint src_width, gint src_height, \ - gdouble src_alpha) \ + gdouble src_alpha, GstCompositorBlendMode mode) \ { \ gint i; \ gint b_alpha; \ \ + /* in source mode we just have to copy over things */ \ + if (mode == COMPOSITOR_BLEND_MODE_SOURCE) { \ + src_alpha = 1.0; \ + } \ + \ /* If it's completely transparent... we just return */ \ if (G_UNLIKELY (src_alpha == 0.0)) { \ GST_INFO ("Fast copy (alpha == 0.0)"); \ @@ -569,7 +593,7 @@ blend_##format_name (GstVideoFrame * srcframe, gint xpos, gint ypos, \ b_dest + comp_xpos + comp_ypos * dest_comp_rowstride, \ src_comp_rowstride, \ dest_comp_rowstride, src_comp_width, src_comp_height, \ - src_alpha); \ + src_alpha, mode); \ \ b_src = GST_VIDEO_FRAME_PLANE_DATA (srcframe, 1); \ b_dest = GST_VIDEO_FRAME_PLANE_DATA (destframe, 1); \ @@ -585,7 +609,7 @@ blend_##format_name (GstVideoFrame * srcframe, gint xpos, gint ypos, \ b_dest + comp_xpos * 2 + comp_ypos * dest_comp_rowstride, \ src_comp_rowstride, \ dest_comp_rowstride, 2 * src_comp_width, src_comp_height, \ - src_alpha); \ + src_alpha, mode); \ } #define NV_YUV_FILL_CHECKER(format_name, MEMSET) \ @@ -711,6 +735,12 @@ blend_##name (GstVideoFrame * srcframe, gint xpos, gint ypos, \ } \ \ dest = dest + bpp * xpos + (ypos * dest_stride); \ + \ + /* in source mode we just have to copy over things */ \ + if (mode == COMPOSITOR_BLEND_MODE_SOURCE) { \ + src_alpha = 1.0; \ + } \ + \ /* If it's completely transparent... we just return */ \ if (G_UNLIKELY (src_alpha == 0.0)) { \ GST_INFO ("Fast copy (alpha == 0.0)"); \ @@ -880,6 +910,12 @@ blend_##name (GstVideoFrame * srcframe, gint xpos, gint ypos, \ } \ \ dest = dest + 2 * xpos + (ypos * dest_stride); \ + \ + /* in source mode we just have to copy over things */ \ + if (mode == COMPOSITOR_BLEND_MODE_SOURCE) { \ + src_alpha = 1.0; \ + } \ + \ /* If it's completely transparent... we just return */ \ if (G_UNLIKELY (src_alpha == 0.0)) { \ GST_INFO ("Fast copy (alpha == 0.0)"); \ diff --git a/gst/compositor/blend.h b/gst/compositor/blend.h index 34d52306a0..710ec7dad0 100644 --- a/gst/compositor/blend.h +++ b/gst/compositor/blend.h @@ -25,15 +25,17 @@ /** * GstCompositorBlendMode: - * @COMPOSITOR_BLEND_MODE_NORMAL: Normal blending - * @COMPOSITOR_BLEND_MODE_ADDITIVE: Alphas are simply added, + * @COMPOSITOR_BLEND_MODE_SOURCE: Copy source + * @COMPOSITOR_BLEND_MODE_OVER: Normal blending + * @COMPOSITOR_BLEND_MODE_ADD: Alphas are simply added, * * The different modes compositor can use for blending. */ typedef enum { - COMPOSITOR_BLEND_MODE_NORMAL, - COMPOSITOR_BLEND_MODE_ADDITIVE, + COMPOSITOR_BLEND_MODE_SOURCE, + COMPOSITOR_BLEND_MODE_OVER, + COMPOSITOR_BLEND_MODE_ADD, } GstCompositorBlendMode; typedef void (*BlendFunction) (GstVideoFrame *srcframe, gint xpos, gint ypos, gdouble src_alpha, GstVideoFrame * destframe, @@ -46,16 +48,11 @@ extern BlendFunction gst_compositor_blend_bgra; #define gst_compositor_blend_ayuv gst_compositor_blend_argb #define gst_compositor_blend_abgr gst_compositor_blend_argb #define gst_compositor_blend_rgba gst_compositor_blend_bgra -#define gst_compositor_blend_ayuv_addition gst_compositor_blend_argb_addition -#define gst_compositor_blend_abgr_addition gst_compositor_blend_argb_addition -#define gst_compositor_blend_rgba_addition gst_compositor_blend_bgra_addition - extern BlendFunction gst_compositor_overlay_argb; extern BlendFunction gst_compositor_overlay_bgra; #define gst_compositor_overlay_ayuv gst_compositor_overlay_argb #define gst_compositor_overlay_abgr gst_compositor_overlay_argb -#define gst_compositor_overlay_abgr gst_compositor_overlay_argb #define gst_compositor_overlay_rgba gst_compositor_overlay_bgra extern BlendFunction gst_compositor_blend_i420; #define gst_compositor_blend_yv12 gst_compositor_blend_i420 diff --git a/gst/compositor/compositor.c b/gst/compositor/compositor.c index d036d68437..453a92e8fa 100644 --- a/gst/compositor/compositor.c +++ b/gst/compositor/compositor.c @@ -122,12 +122,32 @@ static GstStaticPadTemplate sink_factory = GST_STATIC_PAD_TEMPLATE ("sink_%u", static void gst_compositor_child_proxy_init (gpointer g_iface, gpointer iface_data); +#define GST_TYPE_COMPOSITOR_OPERATOR (gst_compositor_operator_get_type()) +static GType +gst_compositor_operator_get_type (void) +{ + static GType compositor_operator_type = 0; + + static const GEnumValue compositor_operator[] = { + {COMPOSITOR_OPERATOR_SOURCE, "Source", "source"}, + {COMPOSITOR_OPERATOR_OVER, "Over", "over"}, + {COMPOSITOR_OPERATOR_ADD, "Add", "add"}, + {0, NULL, NULL}, + }; + + if (!compositor_operator_type) { + compositor_operator_type = + g_enum_register_static ("GstCompositorOperator", compositor_operator); + } + return compositor_operator_type; +} + #define DEFAULT_PAD_XPOS 0 #define DEFAULT_PAD_YPOS 0 #define DEFAULT_PAD_WIDTH 0 #define DEFAULT_PAD_HEIGHT 0 #define DEFAULT_PAD_ALPHA 1.0 -#define DEFAULT_PAD_CROSSFADE_RATIO 0.0 +#define DEFAULT_PAD_OPERATOR COMPOSITOR_OPERATOR_OVER enum { PROP_PAD_0, @@ -136,7 +156,7 @@ enum PROP_PAD_WIDTH, PROP_PAD_HEIGHT, PROP_PAD_ALPHA, - PROP_PAD_CROSSFADE_RATIO, + PROP_PAD_OPERATOR, }; G_DEFINE_TYPE (GstCompositorPad, gst_compositor_pad, @@ -164,8 +184,8 @@ gst_compositor_pad_get_property (GObject * object, guint prop_id, case PROP_PAD_ALPHA: g_value_set_double (value, pad->alpha); break; - case PROP_PAD_CROSSFADE_RATIO: - g_value_set_double (value, pad->crossfade); + case PROP_PAD_OPERATOR: + g_value_set_enum (value, pad->op); break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); @@ -199,10 +219,10 @@ gst_compositor_pad_set_property (GObject * object, guint prop_id, case PROP_PAD_ALPHA: pad->alpha = g_value_get_double (value); break; - case PROP_PAD_CROSSFADE_RATIO: - pad->crossfade = g_value_get_double (value); + case PROP_PAD_OPERATOR: + pad->op = g_value_get_enum (value); gst_video_aggregator_pad_set_needs_alpha (GST_VIDEO_AGGREGATOR_PAD (pad), - pad->crossfade > 0.0); + pad->op == COMPOSITOR_OPERATOR_ADD); break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); @@ -319,7 +339,7 @@ gst_compositor_pad_prepare_frame (GstVideoAggregatorPad * pad, GST_VIDEO_INFO_PAR_D (&vagg->info), &width, &height); if (cpad->alpha == 0.0) { - GST_DEBUG_OBJECT (vagg, "Pad has alpha 0.0, not converting frame"); + GST_DEBUG_OBJECT (pad, "Pad has alpha 0.0, not converting frame"); goto done; } @@ -327,25 +347,16 @@ gst_compositor_pad_prepare_frame (GstVideoAggregatorPad * pad, GST_VIDEO_INFO_WIDTH (&vagg->info), GST_VIDEO_INFO_HEIGHT (&vagg->info)); if (frame_rect.w == 0 || frame_rect.h == 0) { - GST_DEBUG_OBJECT (vagg, "Resulting frame is zero-width or zero-height " + GST_DEBUG_OBJECT (pad, "Resulting frame is zero-width or zero-height " "(w: %i, h: %i), skipping", frame_rect.w, frame_rect.h); goto done; } GST_OBJECT_LOCK (vagg); - /* Check if we are crossfading the pad one way or another */ - l = g_list_find (GST_ELEMENT (vagg)->sinkpads, pad); - if ((l->prev && GST_COMPOSITOR_PAD (l->prev->data)->crossfade > 0.0) || - (GST_COMPOSITOR_PAD (pad)->crossfade > 0.0)) { - GST_DEBUG_OBJECT (pad, "Is being crossfaded with previous pad"); - l = NULL; - } else { - l = l->next; - } - /* Check if this frame is obscured by a higher-zorder frame * TODO: Also skip a frame if it's obscured by a combination of * higher-zorder frames */ + l = g_list_find (GST_ELEMENT (vagg)->sinkpads, pad)->next; for (; l; l = l->next) { GstVideoRectangle frame2_rect; GstVideoAggregatorPad *pad2 = l->data; @@ -468,10 +479,10 @@ gst_compositor_pad_class_init (GstCompositorPadClass * klass) g_param_spec_double ("alpha", "Alpha", "Alpha of the picture", 0.0, 1.0, DEFAULT_PAD_ALPHA, G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); - g_object_class_install_property (gobject_class, PROP_PAD_CROSSFADE_RATIO, - g_param_spec_double ("crossfade-ratio", "Crossfade ratio", - "The crossfade ratio to use while crossfading with the following pad", - 0.0, 1.0, DEFAULT_PAD_CROSSFADE_RATIO, + g_object_class_install_property (gobject_class, PROP_PAD_OPERATOR, + g_param_spec_enum ("operator", "Operator", + "Blending operator to use for blending this pad over the previous ones", + GST_TYPE_COMPOSITOR_OPERATOR, DEFAULT_PAD_OPERATOR, G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); vaggpadclass->prepare_frame = @@ -487,7 +498,7 @@ gst_compositor_pad_init (GstCompositorPad * compo_pad) compo_pad->xpos = DEFAULT_PAD_XPOS; compo_pad->ypos = DEFAULT_PAD_YPOS; compo_pad->alpha = DEFAULT_PAD_ALPHA; - compo_pad->crossfade = DEFAULT_PAD_CROSSFADE_RATIO; + compo_pad->op = DEFAULT_PAD_OPERATOR; } @@ -818,137 +829,6 @@ _negotiated_caps (GstAggregator * agg, GstCaps * caps) return GST_AGGREGATOR_CLASS (parent_class)->negotiated_src_caps (agg, caps); } -/* Fills frame with transparent pixels if @nframe is NULL otherwise copy @frame - * properties and fill @nframes with transparent pixels */ -static GstFlowReturn -gst_compositor_fill_transparent (GstCompositor * self, GstVideoFrame * frame, - GstVideoFrame * nframe) -{ - guint plane, num_planes, height, i; - - if (nframe) { - GstBuffer *cbuffer = gst_buffer_copy_deep (frame->buffer); - - if (!gst_video_frame_map (nframe, &frame->info, cbuffer, GST_MAP_WRITE)) { - GST_WARNING_OBJECT (self, "Could not map output buffer"); - gst_buffer_unref (cbuffer); - return GST_FLOW_ERROR; - } - - /* the last reference is owned by the frame and released once the frame - * is unmapped. We leak it if we don't unref here */ - gst_buffer_unref (cbuffer); - } else { - nframe = frame; - } - - num_planes = GST_VIDEO_FRAME_N_PLANES (nframe); - for (plane = 0; plane < num_planes; ++plane) { - guint8 *pdata; - gsize rowsize, plane_stride; - - pdata = GST_VIDEO_FRAME_PLANE_DATA (nframe, plane); - plane_stride = GST_VIDEO_FRAME_PLANE_STRIDE (nframe, plane); - rowsize = GST_VIDEO_FRAME_COMP_WIDTH (nframe, plane) - * GST_VIDEO_FRAME_COMP_PSTRIDE (nframe, plane); - height = GST_VIDEO_FRAME_COMP_HEIGHT (nframe, plane); - for (i = 0; i < height; ++i) { - memset (pdata, 0, rowsize); - pdata += plane_stride; - } - } - - return GST_FLOW_OK; -} - -/* WITH GST_OBJECT_LOCK !! - * Returns: %TRUE if outframe is allready ready to be used as we are using - * a transparent background and all pads have already been crossfaded - * %FALSE otherwise - */ -static gboolean -gst_compositor_crossfade_frames (GstCompositor * self, GstVideoFrame * outframe) -{ - GList *l; - gboolean all_crossfading = FALSE; - GstVideoAggregator *vagg = GST_VIDEO_AGGREGATOR (self); - - if (self->background == COMPOSITOR_BACKGROUND_TRANSPARENT) { - - all_crossfading = TRUE; - for (l = GST_ELEMENT (self)->sinkpads; l; l = l->next) { - GstCompositorPad *compo_pad = GST_COMPOSITOR_PAD (l->data); - - if (compo_pad->crossfade == 0.0 && l->next && - GST_COMPOSITOR_PAD (l->next->data)->crossfade == 0.0) { - all_crossfading = FALSE; - - break; - } - } - } - - for (l = GST_ELEMENT (self)->sinkpads; l; l = l->next) { - GstVideoAggregatorPad *pad = l->data; - GstCompositorPad *compo_pad = GST_COMPOSITOR_PAD (pad); - GstVideoAggregatorPadClass *pad_class = - GST_VIDEO_AGGREGATOR_PAD_GET_CLASS (compo_pad); - GstVideoFrame *prepared_frame = - gst_video_aggregator_pad_get_prepared_frame (pad); - - if (compo_pad->crossfade > 0.0 && prepared_frame) { - gfloat alpha = compo_pad->crossfade * compo_pad->alpha; - GstVideoAggregatorPad *npad = l->next ? l->next->data : NULL; - GstVideoFrame *next_prepared_frame; - GstVideoFrame nframe; - - next_prepared_frame = - npad ? gst_video_aggregator_pad_get_prepared_frame (npad) : NULL; - - if (!all_crossfading) { - gst_compositor_fill_transparent (self, outframe, &nframe); - } else { - nframe = *outframe; - } - - self->overlay (prepared_frame, - compo_pad->crossfaded ? 0 : compo_pad->xpos, - compo_pad->crossfaded ? 0 : compo_pad->ypos, - alpha, &nframe, COMPOSITOR_BLEND_MODE_ADDITIVE); - - if (npad && next_prepared_frame) { - GstCompositorPad *next_compo_pad = GST_COMPOSITOR_PAD (npad); - - alpha = (1.0 - compo_pad->crossfade) * next_compo_pad->alpha; - self->overlay (next_prepared_frame, next_compo_pad->xpos, - next_compo_pad->ypos, alpha, &nframe, - COMPOSITOR_BLEND_MODE_ADDITIVE); - - /* Replace frame with current frame */ - pad_class->clean_frame (npad, vagg, next_prepared_frame); - if (!all_crossfading) - *next_prepared_frame = nframe; - next_compo_pad->crossfaded = TRUE; - - /* Frame is now consumed, clean it up */ - pad_class->clean_frame (pad, vagg, prepared_frame); - } else { - GST_LOG_OBJECT (self, "Simply fading out as no following pad found"); - pad_class->clean_frame (pad, vagg, prepared_frame); - if (!all_crossfading) - *prepared_frame = nframe; - compo_pad->crossfaded = TRUE; - } - } - } - - if (all_crossfading) - for (l = GST_ELEMENT (self)->sinkpads; l; l = l->next) - GST_COMPOSITOR_PAD (l->data)->crossfaded = FALSE; - - return all_crossfading; -} - static GstFlowReturn gst_compositor_aggregate_frames (GstVideoAggregator * vagg, GstBuffer * outbuf) { @@ -978,28 +858,57 @@ gst_compositor_aggregate_frames (GstVideoAggregator * vagg, GstBuffer * outbuf) self->fill_color (outframe, 240, 128, 128); break; case COMPOSITOR_BACKGROUND_TRANSPARENT: - gst_compositor_fill_transparent (self, outframe, NULL); + { + guint i, plane, num_planes, height; + + num_planes = GST_VIDEO_FRAME_N_PLANES (outframe); + for (plane = 0; plane < num_planes; ++plane) { + guint8 *pdata; + gsize rowsize, plane_stride; + + pdata = GST_VIDEO_FRAME_PLANE_DATA (outframe, plane); + plane_stride = GST_VIDEO_FRAME_PLANE_STRIDE (outframe, plane); + rowsize = GST_VIDEO_FRAME_COMP_WIDTH (outframe, plane) + * GST_VIDEO_FRAME_COMP_PSTRIDE (outframe, plane); + height = GST_VIDEO_FRAME_COMP_HEIGHT (outframe, plane); + for (i = 0; i < height; ++i) { + memset (pdata, 0, rowsize); + pdata += plane_stride; + } + } /* use overlay to keep background transparent */ composite = self->overlay; break; + } } GST_OBJECT_LOCK (vagg); - /* First mix the crossfade frames as required */ - if (!gst_compositor_crossfade_frames (self, outframe)) { - for (l = GST_ELEMENT (vagg)->sinkpads; l; l = l->next) { - GstVideoAggregatorPad *pad = l->data; - GstCompositorPad *compo_pad = GST_COMPOSITOR_PAD (pad); - GstVideoFrame *prepared_frame = - gst_video_aggregator_pad_get_prepared_frame (pad); + for (l = GST_ELEMENT (vagg)->sinkpads; l; l = l->next) { + GstVideoAggregatorPad *pad = l->data; + GstCompositorPad *compo_pad = GST_COMPOSITOR_PAD (pad); + GstVideoFrame *prepared_frame = + gst_video_aggregator_pad_get_prepared_frame (pad); + GstCompositorBlendMode blend_mode = COMPOSITOR_BLEND_MODE_OVER; - if (prepared_frame != NULL) { - composite (prepared_frame, - compo_pad->crossfaded ? 0 : compo_pad->xpos, - compo_pad->crossfaded ? 0 : compo_pad->ypos, compo_pad->alpha, - outframe, COMPOSITOR_BLEND_MODE_NORMAL); - compo_pad->crossfaded = FALSE; - } + switch (compo_pad->op) { + case COMPOSITOR_OPERATOR_SOURCE: + blend_mode = COMPOSITOR_OPERATOR_SOURCE; + break; + case COMPOSITOR_OPERATOR_OVER: + blend_mode = COMPOSITOR_OPERATOR_OVER; + break; + case COMPOSITOR_OPERATOR_ADD: + blend_mode = COMPOSITOR_OPERATOR_ADD; + break; + default: + g_assert_not_reached (); + break; + } + + if (prepared_frame != NULL) { + composite (prepared_frame, + compo_pad->xpos, + compo_pad->ypos, compo_pad->alpha, outframe, blend_mode); } } GST_OBJECT_UNLOCK (vagg); diff --git a/gst/compositor/compositor.h b/gst/compositor/compositor.h index 37ffcb5c4c..5e8d90c82e 100644 --- a/gst/compositor/compositor.h +++ b/gst/compositor/compositor.h @@ -56,7 +56,7 @@ typedef struct _GstCompositorPad GstCompositorPad; typedef struct _GstCompositorPadClass GstCompositorPadClass; /** - * GstcompositorBackground: + * GstCompositorBackground: * @COMPOSITOR_BACKGROUND_CHECKER: checker pattern background * @COMPOSITOR_BACKGROUND_BLACK: solid color black background * @COMPOSITOR_BACKGROUND_WHITE: solid color white background @@ -72,6 +72,28 @@ typedef enum COMPOSITOR_BACKGROUND_TRANSPARENT, } GstCompositorBackground; +/** + * GstCompositorOperator: + * @COMPOSITOR_OPERATOR_SOURCE: Copy the source over the destination, + * without the destination pixels. + * @COMPOSITOR_OPERATOR_OVER: Blend the source over the destination. + * @COMPOSITOR_OPERATOR_ADD: Similar to over but add the source and + * destination alpha. Requires output with alpha + * channel. + * + * The different blending operators that can be used by compositor. + * + * See https://www.cairographics.org/operators/ for some explanation and + * visualizations. + * + */ +typedef enum +{ + COMPOSITOR_OPERATOR_SOURCE, + COMPOSITOR_OPERATOR_OVER, + COMPOSITOR_OPERATOR_ADD, +} GstCompositorOperator; + /** * GstCompositor: * @@ -105,9 +127,8 @@ struct _GstCompositorPad gint xpos, ypos; gint width, height; gdouble alpha; - gdouble crossfade; - gboolean crossfaded; + GstCompositorOperator op; }; struct _GstCompositorPadClass diff --git a/gst/compositor/compositororc.orc b/gst/compositor/compositororc.orc index 66c4dcc730..fea01a473a 100644 --- a/gst/compositor/compositororc.orc +++ b/gst/compositor/compositororc.orc @@ -62,6 +62,36 @@ x4 convwb t, d_wide orl t, t, a_alpha storel d, t +.function compositor_orc_source_argb +.flags 2d +.dest 4 d guint8 +.source 4 s guint8 +.param 2 alpha +.temp 4 t +.temp 4 t2 +.temp 2 tw +.temp 1 tb +.temp 4 a +.temp 8 a_wide +.const 4 a_alpha 0x000000ff +.const 4 a_not_alpha 0xffffff00 + +loadl t, s +shrul t2, t, 24 +convlw tw, t2 +convwb tb, tw +splatbl a, tb +x4 convubw a_wide, a +x4 mullw a_wide, a_wide, alpha +x4 div255w a_wide, a_wide + +andl t, t, a_not_alpha +x4 convwb t2, a_wide +andl t2, t2, a_alpha +orl t, t, t2 + +storel d, t + .function compositor_orc_blend_bgra .flags 2d .dest 4 d guint8 @@ -98,6 +128,36 @@ x4 convwb t, d_wide orl t, t, a_alpha storel d, t +.function compositor_orc_source_bgra +.flags 2d +.dest 4 d guint8 +.source 4 s guint8 +.param 2 alpha +.temp 4 t +.temp 4 t2 +.temp 2 tw +.temp 1 tb +.temp 4 a +.temp 8 a_wide +.const 4 a_alpha 0xff000000 +.const 4 a_not_alpha 0x00ffffff + +loadl t, s +shrul t2, t, 24 +convlw tw, t2 +convwb tb, tw +splatbl a, tb +x4 convubw a_wide, a +x4 mullw a_wide, a_wide, alpha +x4 div255w a_wide, a_wide + +andl t, t, a_not_alpha +x4 convwb t2, a_wide +andl t2, t2, a_alpha +orl t, t, t2 + +storel d, t + .function compositor_orc_overlay_argb .flags 2d .dest 4 d guint8 diff --git a/tests/examples/compositor/crossfade.c b/tests/examples/compositor/crossfade.c index 44090ff556..503503ca9d 100644 --- a/tests/examples/compositor/crossfade.c +++ b/tests/examples/compositor/crossfade.c @@ -21,8 +21,7 @@ /** * Simple crossfade example using the compositor element. * - * Takes a list of uri/path to video files and crossfade them - * for 10 seconds and returns. + * Takes two video files and crossfades them for 10 seconds and returns. */ #include @@ -34,7 +33,6 @@ typedef struct { GstElement *compositor; guint z_order; - gboolean is_last; } VideoInfo; static gchar * @@ -51,24 +49,23 @@ _pad_added_cb (GstElement * decodebin, GstPad * pad, VideoInfo * info) { GstPad *sinkpad = gst_element_get_request_pad (GST_ELEMENT (info->compositor), "sink_%u"); + GstControlSource *control_source; + gboolean is_last = info->z_order == 1; - if (!info->is_last) { - GstControlSource *control_source; + control_source = gst_interpolation_control_source_new (); - control_source = gst_interpolation_control_source_new (); + gst_util_set_object_arg (G_OBJECT (sinkpad), "operator", + info->z_order == 0 ? "source" : "add"); + gst_object_add_control_binding (GST_OBJECT (sinkpad), + gst_direct_control_binding_new_absolute (GST_OBJECT (sinkpad), "alpha", + control_source)); - g_object_set (sinkpad, "crossfade-ratio", 1.0, NULL); - gst_object_add_control_binding (GST_OBJECT (sinkpad), - gst_direct_control_binding_new_absolute (GST_OBJECT (sinkpad), - "crossfade-ratio", control_source)); + g_object_set (control_source, "mode", GST_INTERPOLATION_MODE_LINEAR, NULL); - g_object_set (control_source, "mode", GST_INTERPOLATION_MODE_LINEAR, NULL); - - gst_timed_value_control_source_set (GST_TIMED_VALUE_CONTROL_SOURCE - (control_source), 0, 1.0); - gst_timed_value_control_source_set (GST_TIMED_VALUE_CONTROL_SOURCE - (control_source), 10 * GST_SECOND, 0.0); - } + gst_timed_value_control_source_set (GST_TIMED_VALUE_CONTROL_SOURCE + (control_source), 0, is_last ? 0.0 : 1.0); + gst_timed_value_control_source_set (GST_TIMED_VALUE_CONTROL_SOURCE + (control_source), 10 * GST_SECOND, is_last ? 1.0 : 0.0); g_object_set (sinkpad, "zorder", info->z_order, NULL); gst_pad_link (pad, sinkpad); @@ -84,8 +81,8 @@ main (int argc, char *argv[]) GstElement *compositor, *sink, *pipeline; GstBus *bus; - if (argc < 2) { - g_error ("At least 1 valid video file paths/urls need to " "be provided"); + if (argc != 3) { + g_error ("Need to provide 2 input videos"); return -1; } @@ -101,7 +98,7 @@ main (int argc, char *argv[]) gst_bin_add_many (GST_BIN (pipeline), compositor, sink, NULL); g_assert (gst_element_link (compositor, sink)); - for (i = 1; i < argc; i++) { + for (i = 1; i < 3; i++) { gchar *uri = ensure_uri (argv[i]); VideoInfo *info = g_malloc0 (sizeof (VideoInfo)); GstElement *uridecodebin = gst_element_factory_make ("uridecodebin", NULL); @@ -111,7 +108,6 @@ main (int argc, char *argv[]) info->compositor = compositor; info->z_order = i - 1; - info->is_last = (i == (argc - 1)) && (argc > 2); g_signal_connect (uridecodebin, "pad-added", (GCallback) _pad_added_cb, info); From 4ab4db11ec6f2a52a9ef4612f8c5a26edad1b716 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johan=20Bj=C3=A4reholt?= Date: Tue, 30 Oct 2018 10:05:03 +0100 Subject: [PATCH 379/381] compositor: fix undeclared functions --- gst/compositor/compositororc-dist.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gst/compositor/compositororc-dist.h b/gst/compositor/compositororc-dist.h index ae6f17ac8d..ea8f979f1b 100644 --- a/gst/compositor/compositororc-dist.h +++ b/gst/compositor/compositororc-dist.h @@ -89,6 +89,8 @@ void compositor_orc_overlay_argb (guint8 * ORC_RESTRICT d1, int d1_stride, const void compositor_orc_overlay_argb_addition (guint8 * ORC_RESTRICT d1, int d1_stride, const guint8 * ORC_RESTRICT s1, int s1_stride, int p1, int n, int m); void compositor_orc_overlay_bgra (guint8 * ORC_RESTRICT d1, int d1_stride, const guint8 * ORC_RESTRICT s1, int s1_stride, int p1, int n, int m); void compositor_orc_overlay_bgra_addition (guint8 * ORC_RESTRICT d1, int d1_stride, const guint8 * ORC_RESTRICT s1, int s1_stride, int p1, int n, int m); +void compositor_orc_source_argb (guint8 * ORC_RESTRICT d1, int d1_stride, const guint8 * ORC_RESTRICT s1, int s1_stride, int p1, int n, int m); +void compositor_orc_source_bgra (guint8 * ORC_RESTRICT d1, int d1_stride, const guint8 * ORC_RESTRICT s1, int s1_stride, int p1, int n, int m); #ifdef __cplusplus } From 95436fa66793a14ea5c9d30a08baf316c447f9cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADctor=20Manuel=20J=C3=A1quez=20Leal?= Date: Wed, 31 Oct 2018 19:20:20 +0100 Subject: [PATCH 380/381] compositor: Fix enum type mismatch The variable blend_mode is GstCompositorBlendMode but it is assigned to a GstCompositorOperator enum value. --- gst/compositor/compositor.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/gst/compositor/compositor.c b/gst/compositor/compositor.c index 453a92e8fa..3f43efebb8 100644 --- a/gst/compositor/compositor.c +++ b/gst/compositor/compositor.c @@ -892,13 +892,13 @@ gst_compositor_aggregate_frames (GstVideoAggregator * vagg, GstBuffer * outbuf) switch (compo_pad->op) { case COMPOSITOR_OPERATOR_SOURCE: - blend_mode = COMPOSITOR_OPERATOR_SOURCE; + blend_mode = COMPOSITOR_BLEND_MODE_SOURCE; break; case COMPOSITOR_OPERATOR_OVER: - blend_mode = COMPOSITOR_OPERATOR_OVER; + blend_mode = COMPOSITOR_BLEND_MODE_OVER; break; case COMPOSITOR_OPERATOR_ADD: - blend_mode = COMPOSITOR_OPERATOR_ADD; + blend_mode = COMPOSITOR_BLEND_MODE_ADD; break; default: g_assert_not_reached (); From fb8a503fd473c6eeb9b5565a30a418a5233fb4e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim-Philipp=20M=C3=BCller?= Date: Fri, 2 Nov 2018 20:31:54 +0000 Subject: [PATCH 381/381] compositor: update disted orc backup files --- gst/compositor/compositororc-dist.c | 610 ++++++++++++++++++++++++++++ gst/compositor/compositororc-dist.h | 4 +- 2 files changed, 612 insertions(+), 2 deletions(-) diff --git a/gst/compositor/compositororc-dist.c b/gst/compositor/compositororc-dist.c index 06cbfddead..a931b4d6d5 100644 --- a/gst/compositor/compositororc-dist.c +++ b/gst/compositor/compositororc-dist.c @@ -102,8 +102,12 @@ void compositor_orc_blend_u8 (guint8 * ORC_RESTRICT d1, int d1_stride, const guint8 * ORC_RESTRICT s1, int s1_stride, int p1, int n, int m); void compositor_orc_blend_argb (guint8 * ORC_RESTRICT d1, int d1_stride, const guint8 * ORC_RESTRICT s1, int s1_stride, int p1, int n, int m); +void compositor_orc_source_argb (guint8 * ORC_RESTRICT d1, int d1_stride, + const guint8 * ORC_RESTRICT s1, int s1_stride, int p1, int n, int m); void compositor_orc_blend_bgra (guint8 * ORC_RESTRICT d1, int d1_stride, const guint8 * ORC_RESTRICT s1, int s1_stride, int p1, int n, int m); +void compositor_orc_source_bgra (guint8 * ORC_RESTRICT d1, int d1_stride, + const guint8 * ORC_RESTRICT s1, int s1_stride, int p1, int n, int m); void compositor_orc_overlay_argb (guint8 * ORC_RESTRICT d1, int d1_stride, const guint8 * ORC_RESTRICT s1, int s1_stride, int p1, int n, int m); void compositor_orc_overlay_argb_addition (guint8 * ORC_RESTRICT d1, @@ -931,6 +935,309 @@ compositor_orc_blend_argb (guint8 * ORC_RESTRICT d1, int d1_stride, #endif +/* compositor_orc_source_argb */ +#ifdef DISABLE_ORC +void +compositor_orc_source_argb (guint8 * ORC_RESTRICT d1, int d1_stride, + const guint8 * ORC_RESTRICT s1, int s1_stride, int p1, int n, int m) +{ + int i; + int j; + orc_union32 *ORC_RESTRICT ptr0; + const orc_union32 *ORC_RESTRICT ptr4; + orc_union64 var38; +#if defined(__APPLE__) && __GNUC__ == 4 && __GNUC_MINOR__ == 2 && defined (__i386__) + volatile orc_union32 var39; +#else + orc_union32 var39; +#endif +#if defined(__APPLE__) && __GNUC__ == 4 && __GNUC_MINOR__ == 2 && defined (__i386__) + volatile orc_union32 var40; +#else + orc_union32 var40; +#endif + orc_union32 var41; + orc_union32 var42; + orc_union16 var43; + orc_int8 var44; + orc_union32 var45; + orc_union64 var46; + orc_union64 var47; + orc_union64 var48; + orc_union32 var49; + orc_union32 var50; + orc_union32 var51; + orc_union32 var52; + + for (j = 0; j < m; j++) { + ptr0 = ORC_PTR_OFFSET (d1, d1_stride * j); + ptr4 = ORC_PTR_OFFSET (s1, s1_stride * j); + + /* 6: loadpw */ + var38.x4[0] = p1; + var38.x4[1] = p1; + var38.x4[2] = p1; + var38.x4[3] = p1; + /* 9: loadpl */ + var39.i = 0xffffff00; /* -256 or 2.122e-314f */ + /* 12: loadpl */ + var40.i = 0x000000ff; /* 255 or 1.25987e-321f */ + + for (i = 0; i < n; i++) { + /* 0: loadl */ + var41 = ptr4[i]; + /* 1: shrul */ + var42.i = ((orc_uint32) var41.i) >> 24; + /* 2: convlw */ + var43.i = var42.i; + /* 3: convwb */ + var44 = var43.i; + /* 4: splatbl */ + var45.i = + ((((orc_uint32) var44) & 0xff) << 24) | ((((orc_uint32) var44) & 0xff) + << 16) | ((((orc_uint32) var44) & 0xff) << 8) | (((orc_uint32) var44) + & 0xff); + /* 5: convubw */ + var46.x4[0] = (orc_uint8) var45.x4[0]; + var46.x4[1] = (orc_uint8) var45.x4[1]; + var46.x4[2] = (orc_uint8) var45.x4[2]; + var46.x4[3] = (orc_uint8) var45.x4[3]; + /* 7: mullw */ + var47.x4[0] = (var46.x4[0] * var38.x4[0]) & 0xffff; + var47.x4[1] = (var46.x4[1] * var38.x4[1]) & 0xffff; + var47.x4[2] = (var46.x4[2] * var38.x4[2]) & 0xffff; + var47.x4[3] = (var46.x4[3] * var38.x4[3]) & 0xffff; + /* 8: div255w */ + var48.x4[0] = + ((orc_uint16) (((orc_uint16) (var47.x4[0] + 128)) + + (((orc_uint16) (var47.x4[0] + 128)) >> 8))) >> 8; + var48.x4[1] = + ((orc_uint16) (((orc_uint16) (var47.x4[1] + 128)) + + (((orc_uint16) (var47.x4[1] + 128)) >> 8))) >> 8; + var48.x4[2] = + ((orc_uint16) (((orc_uint16) (var47.x4[2] + 128)) + + (((orc_uint16) (var47.x4[2] + 128)) >> 8))) >> 8; + var48.x4[3] = + ((orc_uint16) (((orc_uint16) (var47.x4[3] + 128)) + + (((orc_uint16) (var47.x4[3] + 128)) >> 8))) >> 8; + /* 10: andl */ + var49.i = var41.i & var39.i; + /* 11: convwb */ + var50.x4[0] = var48.x4[0]; + var50.x4[1] = var48.x4[1]; + var50.x4[2] = var48.x4[2]; + var50.x4[3] = var48.x4[3]; + /* 13: andl */ + var51.i = var50.i & var40.i; + /* 14: orl */ + var52.i = var49.i | var51.i; + /* 15: storel */ + ptr0[i] = var52; + } + } + +} + +#else +static void +_backup_compositor_orc_source_argb (OrcExecutor * ORC_RESTRICT ex) +{ + int i; + int j; + int n = ex->n; + int m = ex->params[ORC_VAR_A1]; + orc_union32 *ORC_RESTRICT ptr0; + const orc_union32 *ORC_RESTRICT ptr4; + orc_union64 var38; +#if defined(__APPLE__) && __GNUC__ == 4 && __GNUC_MINOR__ == 2 && defined (__i386__) + volatile orc_union32 var39; +#else + orc_union32 var39; +#endif +#if defined(__APPLE__) && __GNUC__ == 4 && __GNUC_MINOR__ == 2 && defined (__i386__) + volatile orc_union32 var40; +#else + orc_union32 var40; +#endif + orc_union32 var41; + orc_union32 var42; + orc_union16 var43; + orc_int8 var44; + orc_union32 var45; + orc_union64 var46; + orc_union64 var47; + orc_union64 var48; + orc_union32 var49; + orc_union32 var50; + orc_union32 var51; + orc_union32 var52; + + for (j = 0; j < m; j++) { + ptr0 = ORC_PTR_OFFSET (ex->arrays[0], ex->params[0] * j); + ptr4 = ORC_PTR_OFFSET (ex->arrays[4], ex->params[4] * j); + + /* 6: loadpw */ + var38.x4[0] = ex->params[24]; + var38.x4[1] = ex->params[24]; + var38.x4[2] = ex->params[24]; + var38.x4[3] = ex->params[24]; + /* 9: loadpl */ + var39.i = 0xffffff00; /* -256 or 2.122e-314f */ + /* 12: loadpl */ + var40.i = 0x000000ff; /* 255 or 1.25987e-321f */ + + for (i = 0; i < n; i++) { + /* 0: loadl */ + var41 = ptr4[i]; + /* 1: shrul */ + var42.i = ((orc_uint32) var41.i) >> 24; + /* 2: convlw */ + var43.i = var42.i; + /* 3: convwb */ + var44 = var43.i; + /* 4: splatbl */ + var45.i = + ((((orc_uint32) var44) & 0xff) << 24) | ((((orc_uint32) var44) & 0xff) + << 16) | ((((orc_uint32) var44) & 0xff) << 8) | (((orc_uint32) var44) + & 0xff); + /* 5: convubw */ + var46.x4[0] = (orc_uint8) var45.x4[0]; + var46.x4[1] = (orc_uint8) var45.x4[1]; + var46.x4[2] = (orc_uint8) var45.x4[2]; + var46.x4[3] = (orc_uint8) var45.x4[3]; + /* 7: mullw */ + var47.x4[0] = (var46.x4[0] * var38.x4[0]) & 0xffff; + var47.x4[1] = (var46.x4[1] * var38.x4[1]) & 0xffff; + var47.x4[2] = (var46.x4[2] * var38.x4[2]) & 0xffff; + var47.x4[3] = (var46.x4[3] * var38.x4[3]) & 0xffff; + /* 8: div255w */ + var48.x4[0] = + ((orc_uint16) (((orc_uint16) (var47.x4[0] + 128)) + + (((orc_uint16) (var47.x4[0] + 128)) >> 8))) >> 8; + var48.x4[1] = + ((orc_uint16) (((orc_uint16) (var47.x4[1] + 128)) + + (((orc_uint16) (var47.x4[1] + 128)) >> 8))) >> 8; + var48.x4[2] = + ((orc_uint16) (((orc_uint16) (var47.x4[2] + 128)) + + (((orc_uint16) (var47.x4[2] + 128)) >> 8))) >> 8; + var48.x4[3] = + ((orc_uint16) (((orc_uint16) (var47.x4[3] + 128)) + + (((orc_uint16) (var47.x4[3] + 128)) >> 8))) >> 8; + /* 10: andl */ + var49.i = var41.i & var39.i; + /* 11: convwb */ + var50.x4[0] = var48.x4[0]; + var50.x4[1] = var48.x4[1]; + var50.x4[2] = var48.x4[2]; + var50.x4[3] = var48.x4[3]; + /* 13: andl */ + var51.i = var50.i & var40.i; + /* 14: orl */ + var52.i = var49.i | var51.i; + /* 15: storel */ + ptr0[i] = var52; + } + } + +} + +void +compositor_orc_source_argb (guint8 * ORC_RESTRICT d1, int d1_stride, + const guint8 * ORC_RESTRICT s1, int s1_stride, int p1, int n, int m) +{ + OrcExecutor _ex, *ex = &_ex; + static volatile int p_inited = 0; + static OrcCode *c = 0; + void (*func) (OrcExecutor *); + + if (!p_inited) { + orc_once_mutex_lock (); + if (!p_inited) { + OrcProgram *p; + +#if 1 + static const orc_uint8 bc[] = { + 1, 7, 9, 26, 99, 111, 109, 112, 111, 115, 105, 116, 111, 114, 95, 111, + 114, 99, 95, 115, 111, 117, 114, 99, 101, 95, 97, 114, 103, 98, 11, 4, + 4, 12, 4, 4, 14, 4, 255, 0, 0, 0, 14, 4, 0, 255, 255, 255, + 14, 4, 24, 0, 0, 0, 16, 2, 20, 4, 20, 4, 20, 2, 20, 1, + 20, 4, 20, 8, 113, 32, 4, 126, 33, 32, 18, 163, 34, 33, 157, 35, + 34, 152, 36, 35, 21, 2, 150, 37, 36, 21, 2, 89, 37, 37, 24, 21, + 2, 80, 37, 37, 106, 32, 32, 17, 21, 2, 157, 33, 37, 106, 33, 33, + 16, 123, 32, 32, 33, 128, 0, 32, 2, 0, + }; + p = orc_program_new_from_static_bytecode (bc); + orc_program_set_backup_function (p, _backup_compositor_orc_source_argb); +#else + p = orc_program_new (); + orc_program_set_2d (p); + orc_program_set_name (p, "compositor_orc_source_argb"); + orc_program_set_backup_function (p, _backup_compositor_orc_source_argb); + orc_program_add_destination (p, 4, "d1"); + orc_program_add_source (p, 4, "s1"); + orc_program_add_constant (p, 4, 0x000000ff, "c1"); + orc_program_add_constant (p, 4, 0xffffff00, "c2"); + orc_program_add_constant (p, 4, 0x00000018, "c3"); + orc_program_add_parameter (p, 2, "p1"); + orc_program_add_temporary (p, 4, "t1"); + orc_program_add_temporary (p, 4, "t2"); + orc_program_add_temporary (p, 2, "t3"); + orc_program_add_temporary (p, 1, "t4"); + orc_program_add_temporary (p, 4, "t5"); + orc_program_add_temporary (p, 8, "t6"); + + orc_program_append_2 (p, "loadl", 0, ORC_VAR_T1, ORC_VAR_S1, ORC_VAR_D1, + ORC_VAR_D1); + orc_program_append_2 (p, "shrul", 0, ORC_VAR_T2, ORC_VAR_T1, ORC_VAR_C3, + ORC_VAR_D1); + orc_program_append_2 (p, "convlw", 0, ORC_VAR_T3, ORC_VAR_T2, ORC_VAR_D1, + ORC_VAR_D1); + orc_program_append_2 (p, "convwb", 0, ORC_VAR_T4, ORC_VAR_T3, ORC_VAR_D1, + ORC_VAR_D1); + orc_program_append_2 (p, "splatbl", 0, ORC_VAR_T5, ORC_VAR_T4, ORC_VAR_D1, + ORC_VAR_D1); + orc_program_append_2 (p, "convubw", 2, ORC_VAR_T6, ORC_VAR_T5, ORC_VAR_D1, + ORC_VAR_D1); + orc_program_append_2 (p, "mullw", 2, ORC_VAR_T6, ORC_VAR_T6, ORC_VAR_P1, + ORC_VAR_D1); + orc_program_append_2 (p, "div255w", 2, ORC_VAR_T6, ORC_VAR_T6, ORC_VAR_D1, + ORC_VAR_D1); + orc_program_append_2 (p, "andl", 0, ORC_VAR_T1, ORC_VAR_T1, ORC_VAR_C2, + ORC_VAR_D1); + orc_program_append_2 (p, "convwb", 2, ORC_VAR_T2, ORC_VAR_T6, ORC_VAR_D1, + ORC_VAR_D1); + orc_program_append_2 (p, "andl", 0, ORC_VAR_T2, ORC_VAR_T2, ORC_VAR_C1, + ORC_VAR_D1); + orc_program_append_2 (p, "orl", 0, ORC_VAR_T1, ORC_VAR_T1, ORC_VAR_T2, + ORC_VAR_D1); + orc_program_append_2 (p, "storel", 0, ORC_VAR_D1, ORC_VAR_T1, ORC_VAR_D1, + ORC_VAR_D1); +#endif + + orc_program_compile (p); + c = orc_program_take_code (p); + orc_program_free (p); + } + p_inited = TRUE; + orc_once_mutex_unlock (); + } + ex->arrays[ORC_VAR_A2] = c; + ex->program = 0; + + ex->n = n; + ORC_EXECUTOR_M (ex) = m; + ex->arrays[ORC_VAR_D1] = d1; + ex->params[ORC_VAR_D1] = d1_stride; + ex->arrays[ORC_VAR_S1] = (void *) s1; + ex->params[ORC_VAR_S1] = s1_stride; + ex->params[ORC_VAR_P1] = p1; + + func = c->exec; + func (ex); +} +#endif + + /* compositor_orc_blend_bgra */ #ifdef DISABLE_ORC void @@ -1315,6 +1622,309 @@ compositor_orc_blend_bgra (guint8 * ORC_RESTRICT d1, int d1_stride, #endif +/* compositor_orc_source_bgra */ +#ifdef DISABLE_ORC +void +compositor_orc_source_bgra (guint8 * ORC_RESTRICT d1, int d1_stride, + const guint8 * ORC_RESTRICT s1, int s1_stride, int p1, int n, int m) +{ + int i; + int j; + orc_union32 *ORC_RESTRICT ptr0; + const orc_union32 *ORC_RESTRICT ptr4; + orc_union64 var38; +#if defined(__APPLE__) && __GNUC__ == 4 && __GNUC_MINOR__ == 2 && defined (__i386__) + volatile orc_union32 var39; +#else + orc_union32 var39; +#endif +#if defined(__APPLE__) && __GNUC__ == 4 && __GNUC_MINOR__ == 2 && defined (__i386__) + volatile orc_union32 var40; +#else + orc_union32 var40; +#endif + orc_union32 var41; + orc_union32 var42; + orc_union16 var43; + orc_int8 var44; + orc_union32 var45; + orc_union64 var46; + orc_union64 var47; + orc_union64 var48; + orc_union32 var49; + orc_union32 var50; + orc_union32 var51; + orc_union32 var52; + + for (j = 0; j < m; j++) { + ptr0 = ORC_PTR_OFFSET (d1, d1_stride * j); + ptr4 = ORC_PTR_OFFSET (s1, s1_stride * j); + + /* 6: loadpw */ + var38.x4[0] = p1; + var38.x4[1] = p1; + var38.x4[2] = p1; + var38.x4[3] = p1; + /* 9: loadpl */ + var39.i = 0x00ffffff; /* 16777215 or 8.28905e-317f */ + /* 12: loadpl */ + var40.i = 0xff000000; /* -16777216 or 2.11371e-314f */ + + for (i = 0; i < n; i++) { + /* 0: loadl */ + var41 = ptr4[i]; + /* 1: shrul */ + var42.i = ((orc_uint32) var41.i) >> 24; + /* 2: convlw */ + var43.i = var42.i; + /* 3: convwb */ + var44 = var43.i; + /* 4: splatbl */ + var45.i = + ((((orc_uint32) var44) & 0xff) << 24) | ((((orc_uint32) var44) & 0xff) + << 16) | ((((orc_uint32) var44) & 0xff) << 8) | (((orc_uint32) var44) + & 0xff); + /* 5: convubw */ + var46.x4[0] = (orc_uint8) var45.x4[0]; + var46.x4[1] = (orc_uint8) var45.x4[1]; + var46.x4[2] = (orc_uint8) var45.x4[2]; + var46.x4[3] = (orc_uint8) var45.x4[3]; + /* 7: mullw */ + var47.x4[0] = (var46.x4[0] * var38.x4[0]) & 0xffff; + var47.x4[1] = (var46.x4[1] * var38.x4[1]) & 0xffff; + var47.x4[2] = (var46.x4[2] * var38.x4[2]) & 0xffff; + var47.x4[3] = (var46.x4[3] * var38.x4[3]) & 0xffff; + /* 8: div255w */ + var48.x4[0] = + ((orc_uint16) (((orc_uint16) (var47.x4[0] + 128)) + + (((orc_uint16) (var47.x4[0] + 128)) >> 8))) >> 8; + var48.x4[1] = + ((orc_uint16) (((orc_uint16) (var47.x4[1] + 128)) + + (((orc_uint16) (var47.x4[1] + 128)) >> 8))) >> 8; + var48.x4[2] = + ((orc_uint16) (((orc_uint16) (var47.x4[2] + 128)) + + (((orc_uint16) (var47.x4[2] + 128)) >> 8))) >> 8; + var48.x4[3] = + ((orc_uint16) (((orc_uint16) (var47.x4[3] + 128)) + + (((orc_uint16) (var47.x4[3] + 128)) >> 8))) >> 8; + /* 10: andl */ + var49.i = var41.i & var39.i; + /* 11: convwb */ + var50.x4[0] = var48.x4[0]; + var50.x4[1] = var48.x4[1]; + var50.x4[2] = var48.x4[2]; + var50.x4[3] = var48.x4[3]; + /* 13: andl */ + var51.i = var50.i & var40.i; + /* 14: orl */ + var52.i = var49.i | var51.i; + /* 15: storel */ + ptr0[i] = var52; + } + } + +} + +#else +static void +_backup_compositor_orc_source_bgra (OrcExecutor * ORC_RESTRICT ex) +{ + int i; + int j; + int n = ex->n; + int m = ex->params[ORC_VAR_A1]; + orc_union32 *ORC_RESTRICT ptr0; + const orc_union32 *ORC_RESTRICT ptr4; + orc_union64 var38; +#if defined(__APPLE__) && __GNUC__ == 4 && __GNUC_MINOR__ == 2 && defined (__i386__) + volatile orc_union32 var39; +#else + orc_union32 var39; +#endif +#if defined(__APPLE__) && __GNUC__ == 4 && __GNUC_MINOR__ == 2 && defined (__i386__) + volatile orc_union32 var40; +#else + orc_union32 var40; +#endif + orc_union32 var41; + orc_union32 var42; + orc_union16 var43; + orc_int8 var44; + orc_union32 var45; + orc_union64 var46; + orc_union64 var47; + orc_union64 var48; + orc_union32 var49; + orc_union32 var50; + orc_union32 var51; + orc_union32 var52; + + for (j = 0; j < m; j++) { + ptr0 = ORC_PTR_OFFSET (ex->arrays[0], ex->params[0] * j); + ptr4 = ORC_PTR_OFFSET (ex->arrays[4], ex->params[4] * j); + + /* 6: loadpw */ + var38.x4[0] = ex->params[24]; + var38.x4[1] = ex->params[24]; + var38.x4[2] = ex->params[24]; + var38.x4[3] = ex->params[24]; + /* 9: loadpl */ + var39.i = 0x00ffffff; /* 16777215 or 8.28905e-317f */ + /* 12: loadpl */ + var40.i = 0xff000000; /* -16777216 or 2.11371e-314f */ + + for (i = 0; i < n; i++) { + /* 0: loadl */ + var41 = ptr4[i]; + /* 1: shrul */ + var42.i = ((orc_uint32) var41.i) >> 24; + /* 2: convlw */ + var43.i = var42.i; + /* 3: convwb */ + var44 = var43.i; + /* 4: splatbl */ + var45.i = + ((((orc_uint32) var44) & 0xff) << 24) | ((((orc_uint32) var44) & 0xff) + << 16) | ((((orc_uint32) var44) & 0xff) << 8) | (((orc_uint32) var44) + & 0xff); + /* 5: convubw */ + var46.x4[0] = (orc_uint8) var45.x4[0]; + var46.x4[1] = (orc_uint8) var45.x4[1]; + var46.x4[2] = (orc_uint8) var45.x4[2]; + var46.x4[3] = (orc_uint8) var45.x4[3]; + /* 7: mullw */ + var47.x4[0] = (var46.x4[0] * var38.x4[0]) & 0xffff; + var47.x4[1] = (var46.x4[1] * var38.x4[1]) & 0xffff; + var47.x4[2] = (var46.x4[2] * var38.x4[2]) & 0xffff; + var47.x4[3] = (var46.x4[3] * var38.x4[3]) & 0xffff; + /* 8: div255w */ + var48.x4[0] = + ((orc_uint16) (((orc_uint16) (var47.x4[0] + 128)) + + (((orc_uint16) (var47.x4[0] + 128)) >> 8))) >> 8; + var48.x4[1] = + ((orc_uint16) (((orc_uint16) (var47.x4[1] + 128)) + + (((orc_uint16) (var47.x4[1] + 128)) >> 8))) >> 8; + var48.x4[2] = + ((orc_uint16) (((orc_uint16) (var47.x4[2] + 128)) + + (((orc_uint16) (var47.x4[2] + 128)) >> 8))) >> 8; + var48.x4[3] = + ((orc_uint16) (((orc_uint16) (var47.x4[3] + 128)) + + (((orc_uint16) (var47.x4[3] + 128)) >> 8))) >> 8; + /* 10: andl */ + var49.i = var41.i & var39.i; + /* 11: convwb */ + var50.x4[0] = var48.x4[0]; + var50.x4[1] = var48.x4[1]; + var50.x4[2] = var48.x4[2]; + var50.x4[3] = var48.x4[3]; + /* 13: andl */ + var51.i = var50.i & var40.i; + /* 14: orl */ + var52.i = var49.i | var51.i; + /* 15: storel */ + ptr0[i] = var52; + } + } + +} + +void +compositor_orc_source_bgra (guint8 * ORC_RESTRICT d1, int d1_stride, + const guint8 * ORC_RESTRICT s1, int s1_stride, int p1, int n, int m) +{ + OrcExecutor _ex, *ex = &_ex; + static volatile int p_inited = 0; + static OrcCode *c = 0; + void (*func) (OrcExecutor *); + + if (!p_inited) { + orc_once_mutex_lock (); + if (!p_inited) { + OrcProgram *p; + +#if 1 + static const orc_uint8 bc[] = { + 1, 7, 9, 26, 99, 111, 109, 112, 111, 115, 105, 116, 111, 114, 95, 111, + 114, 99, 95, 115, 111, 117, 114, 99, 101, 95, 98, 103, 114, 97, 11, 4, + 4, 12, 4, 4, 14, 4, 0, 0, 0, 255, 14, 4, 255, 255, 255, 0, + 14, 4, 24, 0, 0, 0, 16, 2, 20, 4, 20, 4, 20, 2, 20, 1, + 20, 4, 20, 8, 113, 32, 4, 126, 33, 32, 18, 163, 34, 33, 157, 35, + 34, 152, 36, 35, 21, 2, 150, 37, 36, 21, 2, 89, 37, 37, 24, 21, + 2, 80, 37, 37, 106, 32, 32, 17, 21, 2, 157, 33, 37, 106, 33, 33, + 16, 123, 32, 32, 33, 128, 0, 32, 2, 0, + }; + p = orc_program_new_from_static_bytecode (bc); + orc_program_set_backup_function (p, _backup_compositor_orc_source_bgra); +#else + p = orc_program_new (); + orc_program_set_2d (p); + orc_program_set_name (p, "compositor_orc_source_bgra"); + orc_program_set_backup_function (p, _backup_compositor_orc_source_bgra); + orc_program_add_destination (p, 4, "d1"); + orc_program_add_source (p, 4, "s1"); + orc_program_add_constant (p, 4, 0xff000000, "c1"); + orc_program_add_constant (p, 4, 0x00ffffff, "c2"); + orc_program_add_constant (p, 4, 0x00000018, "c3"); + orc_program_add_parameter (p, 2, "p1"); + orc_program_add_temporary (p, 4, "t1"); + orc_program_add_temporary (p, 4, "t2"); + orc_program_add_temporary (p, 2, "t3"); + orc_program_add_temporary (p, 1, "t4"); + orc_program_add_temporary (p, 4, "t5"); + orc_program_add_temporary (p, 8, "t6"); + + orc_program_append_2 (p, "loadl", 0, ORC_VAR_T1, ORC_VAR_S1, ORC_VAR_D1, + ORC_VAR_D1); + orc_program_append_2 (p, "shrul", 0, ORC_VAR_T2, ORC_VAR_T1, ORC_VAR_C3, + ORC_VAR_D1); + orc_program_append_2 (p, "convlw", 0, ORC_VAR_T3, ORC_VAR_T2, ORC_VAR_D1, + ORC_VAR_D1); + orc_program_append_2 (p, "convwb", 0, ORC_VAR_T4, ORC_VAR_T3, ORC_VAR_D1, + ORC_VAR_D1); + orc_program_append_2 (p, "splatbl", 0, ORC_VAR_T5, ORC_VAR_T4, ORC_VAR_D1, + ORC_VAR_D1); + orc_program_append_2 (p, "convubw", 2, ORC_VAR_T6, ORC_VAR_T5, ORC_VAR_D1, + ORC_VAR_D1); + orc_program_append_2 (p, "mullw", 2, ORC_VAR_T6, ORC_VAR_T6, ORC_VAR_P1, + ORC_VAR_D1); + orc_program_append_2 (p, "div255w", 2, ORC_VAR_T6, ORC_VAR_T6, ORC_VAR_D1, + ORC_VAR_D1); + orc_program_append_2 (p, "andl", 0, ORC_VAR_T1, ORC_VAR_T1, ORC_VAR_C2, + ORC_VAR_D1); + orc_program_append_2 (p, "convwb", 2, ORC_VAR_T2, ORC_VAR_T6, ORC_VAR_D1, + ORC_VAR_D1); + orc_program_append_2 (p, "andl", 0, ORC_VAR_T2, ORC_VAR_T2, ORC_VAR_C1, + ORC_VAR_D1); + orc_program_append_2 (p, "orl", 0, ORC_VAR_T1, ORC_VAR_T1, ORC_VAR_T2, + ORC_VAR_D1); + orc_program_append_2 (p, "storel", 0, ORC_VAR_D1, ORC_VAR_T1, ORC_VAR_D1, + ORC_VAR_D1); +#endif + + orc_program_compile (p); + c = orc_program_take_code (p); + orc_program_free (p); + } + p_inited = TRUE; + orc_once_mutex_unlock (); + } + ex->arrays[ORC_VAR_A2] = c; + ex->program = 0; + + ex->n = n; + ORC_EXECUTOR_M (ex) = m; + ex->arrays[ORC_VAR_D1] = d1; + ex->params[ORC_VAR_D1] = d1_stride; + ex->arrays[ORC_VAR_S1] = (void *) s1; + ex->params[ORC_VAR_S1] = s1_stride; + ex->params[ORC_VAR_P1] = p1; + + func = c->exec; + func (ex); +} +#endif + + /* compositor_orc_overlay_argb */ #ifdef DISABLE_ORC void diff --git a/gst/compositor/compositororc-dist.h b/gst/compositor/compositororc-dist.h index ea8f979f1b..75b6fb76a3 100644 --- a/gst/compositor/compositororc-dist.h +++ b/gst/compositor/compositororc-dist.h @@ -84,13 +84,13 @@ void compositor_orc_splat_u32 (guint32 * ORC_RESTRICT d1, int p1, int n); void compositor_orc_memcpy_u32 (guint32 * ORC_RESTRICT d1, const guint32 * ORC_RESTRICT s1, int n); void compositor_orc_blend_u8 (guint8 * ORC_RESTRICT d1, int d1_stride, const guint8 * ORC_RESTRICT s1, int s1_stride, int p1, int n, int m); void compositor_orc_blend_argb (guint8 * ORC_RESTRICT d1, int d1_stride, const guint8 * ORC_RESTRICT s1, int s1_stride, int p1, int n, int m); +void compositor_orc_source_argb (guint8 * ORC_RESTRICT d1, int d1_stride, const guint8 * ORC_RESTRICT s1, int s1_stride, int p1, int n, int m); void compositor_orc_blend_bgra (guint8 * ORC_RESTRICT d1, int d1_stride, const guint8 * ORC_RESTRICT s1, int s1_stride, int p1, int n, int m); +void compositor_orc_source_bgra (guint8 * ORC_RESTRICT d1, int d1_stride, const guint8 * ORC_RESTRICT s1, int s1_stride, int p1, int n, int m); void compositor_orc_overlay_argb (guint8 * ORC_RESTRICT d1, int d1_stride, const guint8 * ORC_RESTRICT s1, int s1_stride, int p1, int n, int m); void compositor_orc_overlay_argb_addition (guint8 * ORC_RESTRICT d1, int d1_stride, const guint8 * ORC_RESTRICT s1, int s1_stride, int p1, int n, int m); void compositor_orc_overlay_bgra (guint8 * ORC_RESTRICT d1, int d1_stride, const guint8 * ORC_RESTRICT s1, int s1_stride, int p1, int n, int m); void compositor_orc_overlay_bgra_addition (guint8 * ORC_RESTRICT d1, int d1_stride, const guint8 * ORC_RESTRICT s1, int s1_stride, int p1, int n, int m); -void compositor_orc_source_argb (guint8 * ORC_RESTRICT d1, int d1_stride, const guint8 * ORC_RESTRICT s1, int s1_stride, int p1, int n, int m); -void compositor_orc_source_bgra (guint8 * ORC_RESTRICT d1, int d1_stride, const guint8 * ORC_RESTRICT s1, int s1_stride, int p1, int n, int m); #ifdef __cplusplus }