diff --git a/sys/d3d11/gstd3d11compositor.c b/sys/d3d11/gstd3d11compositor.c new file mode 100644 index 0000000000..37cab96bed --- /dev/null +++ b/sys/d3d11/gstd3d11compositor.c @@ -0,0 +1,2221 @@ +/* + * GStreamer + * Copyright (C) 2004, 2008 Wim Taymans + * Copyright (C) 2010 Sebastian Dröge + * Copyright (C) 2014 Mathieu Duponchelle + * Copyright (C) 2014 Thibault Saunier + * Copyright (C) 2020 Seungha Yang + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "gstd3d11_fwd.h" +#include "gstd3d11compositor.h" +#include "gstd3d11device.h" +#include "gstd3d11memory.h" +#include "gstd3d11bufferpool.h" +#include "gstd3d11utils.h" +#include "gstd3d11format.h" +#include "gstd3d11colorconverter.h" +#include "gstd3d11shader.h" +#include + +GST_DEBUG_CATEGORY_EXTERN (gst_d3d11_compositor_debug); +#define GST_CAT_DEFAULT gst_d3d11_compositor_debug + +/** + * GstD3D11CompositorBlendOperation: + * @GST_D3D11_COMPOSITOR_BLEND_OP_ADD: + * Add source 1 and source 2 + * @GST_D3D11_COMPOSITOR_BLEND_OP_SUBTRACT: + * Subtract source 1 from source 2 + * @GST_D3D11_COMPOSITOR_BLEND_OP_REV_SUBTRACT: + * Subtract source 2 from source 1 + * @GST_D3D11_COMPOSITOR_BLEND_OP_MIN: + * Find the minimum of source 1 and source 2 + * @GST_D3D11_COMPOSITOR_BLEND_OP_MAX: + * Find the maximum of source 1 and source 2 + */ +GType +gst_d3d11_compositor_blend_operation_get_type (void) +{ + static GType blend_operation_type = 0; + + static const GEnumValue blend_operator[] = { + {GST_D3D11_COMPOSITOR_BLEND_OP_ADD, "Add source and background", + "add"}, + {GST_D3D11_COMPOSITOR_BLEND_OP_SUBTRACT, + "Subtract source from background", + "subtract"}, + {GST_D3D11_COMPOSITOR_BLEND_OP_REV_SUBTRACT, + "Subtract background from source", + "rev-subtract"}, + {GST_D3D11_COMPOSITOR_BLEND_OP_MIN, + "Minimum of source and background", "min"}, + {GST_D3D11_COMPOSITOR_BLEND_OP_MAX, + "Maximum of source and background", "max"}, + {0, NULL, NULL}, + }; + + if (!blend_operation_type) { + blend_operation_type = + g_enum_register_static ("GstD3D11CompositorBlendOperation", + blend_operator); + } + return blend_operation_type; +} + +static GstD3D11CompositorBlendOperation +gst_d3d11_compositor_blend_operation_from_native (D3D11_BLEND_OP blend_op) +{ + switch (blend_op) { + case D3D11_BLEND_OP_ADD: + return GST_D3D11_COMPOSITOR_BLEND_OP_ADD; + case D3D11_BLEND_OP_SUBTRACT: + return GST_D3D11_COMPOSITOR_BLEND_OP_SUBTRACT; + case D3D11_BLEND_OP_REV_SUBTRACT: + return GST_D3D11_COMPOSITOR_BLEND_OP_REV_SUBTRACT; + case D3D11_BLEND_OP_MIN: + return GST_D3D11_COMPOSITOR_BLEND_OP_MIN; + case D3D11_BLEND_OP_MAX: + return GST_D3D11_COMPOSITOR_BLEND_OP_MAX; + default: + g_assert_not_reached (); + break; + } + + return GST_D3D11_COMPOSITOR_BLEND_OP_ADD; +} + +static D3D11_BLEND_OP +gst_d3d11_compositor_blend_operation_to_native (GstD3D11CompositorBlendOperation + op) +{ + switch (op) { + case GST_D3D11_COMPOSITOR_BLEND_OP_ADD: + return D3D11_BLEND_OP_ADD; + case GST_D3D11_COMPOSITOR_BLEND_OP_SUBTRACT: + return D3D11_BLEND_OP_SUBTRACT; + case GST_D3D11_COMPOSITOR_BLEND_OP_REV_SUBTRACT: + return D3D11_BLEND_OP_REV_SUBTRACT; + case GST_D3D11_COMPOSITOR_BLEND_OP_MIN: + return D3D11_BLEND_OP_MIN; + case GST_D3D11_COMPOSITOR_BLEND_OP_MAX: + return D3D11_BLEND_OP_MAX; + default: + g_assert_not_reached (); + break; + } + + return D3D11_BLEND_OP_ADD; +} + +/** + * GstD3D11CompositorBlend: + * @GST_D3D11_COMPOSITOR_BLEND_ZERO: + * The blend factor is (0, 0, 0, 0). No pre-blend operation. + * @GST_D3D11_COMPOSITOR_BLEND_ONE: + * The blend factor is (1, 1, 1, 1). No pre-blend operation. + * @GST_D3D11_COMPOSITOR_BLEND_SRC_COLOR: + * The blend factor is (Rs, Gs, Bs, As), + * that is color data (RGB) from a pixel shader. No pre-blend operation. + * @GST_D3D11_COMPOSITOR_BLEND_INV_SRC_COLOR: + * The blend factor is (1 - Rs, 1 - Gs, 1 - Bs, 1 - As), + * that is color data (RGB) from a pixel shader. + * The pre-blend operation inverts the data, generating 1 - RGB. + * @GST_D3D11_COMPOSITOR_BLEND_SRC_ALPHA: + * The blend factor is (As, As, As, As), + * that is alpha data (A) from a pixel shader. No pre-blend operation. + * @GST_D3D11_COMPOSITOR_BLEND_INV_SRC_ALPHA: + * The blend factor is ( 1 - As, 1 - As, 1 - As, 1 - As), + * that is alpha data (A) from a pixel shader. + * The pre-blend operation inverts the data, generating 1 - A. + * @GST_D3D11_COMPOSITOR_BLEND_DEST_ALPHA: + * The blend factor is (Ad, Ad, Ad, Ad), + * that is alpha data from a render target. No pre-blend operation. + * @GST_D3D11_COMPOSITOR_BLEND_INV_DEST_ALPHA: + * The blend factor is (1 - Ad, 1 - Ad, 1 - Ad, 1 - Ad), + * that is alpha data from a render target. + * The pre-blend operation inverts the data, generating 1 - A. + * @GST_D3D11_COMPOSITOR_BLEND_DEST_COLOR: + * The blend factor is (Rd, Gd, Bd, Ad), + * that is color data from a render target. No pre-blend operation. + * @GST_D3D11_COMPOSITOR_BLEND_INV_DEST_COLOR: + * The blend factor is (1 - Rd, 1 - Gd, 1 - Bd, 1 - Ad), + * that is color data from a render target. + * The pre-blend operation inverts the data, generating 1 - RGB. + * @GST_D3D11_COMPOSITOR_BLEND_SRC_ALPHA_SAT: + * The blend factor is (f, f, f, 1); where f = min(As, 1 - Ad). + * The pre-blend operation clamps the data to 1 or less. + * @GST_D3D11_COMPOSITOR_BLEND_BLEND_FACTOR: + * The blend factor is the blend factor set with + * ID3D11DeviceContext::OMSetBlendState. No pre-blend operation. + * @GST_D3D11_COMPOSITOR_BLEND_INV_BLEND_FACTOR: + * The blend factor is the blend factor set with + * ID3D11DeviceContext::OMSetBlendState. + * The pre-blend operation inverts the blend factor, + * generating 1 - blend_factor. + */ +GType +gst_d3d11_compositor_blend_get_type (void) +{ + static GType blend_type = 0; + + static const GEnumValue blend[] = { + {GST_D3D11_COMPOSITOR_BLEND_ZERO, + "The blend factor is (0, 0, 0, 0)", "zero"}, + {GST_D3D11_COMPOSITOR_BLEND_ONE, + "The blend factor is (1, 1, 1, 1)", "one"}, + {GST_D3D11_COMPOSITOR_BLEND_SRC_COLOR, + "The blend factor is (Rs, Gs, Bs, As)", "src-color"}, + {GST_D3D11_COMPOSITOR_BLEND_INV_SRC_COLOR, + "The blend factor is (1 - Rs, 1 - Gs, 1 - Bs, 1 - As)", + "inv-src-color"}, + {GST_D3D11_COMPOSITOR_BLEND_SRC_ALPHA, + "The blend factor is (As, As, As, As)", "src-alpha"}, + {GST_D3D11_COMPOSITOR_BLEND_INV_SRC_ALPHA, + "The blend factor is (1 - As, 1 - As, 1 - As, 1 - As)", + "inv-src-alpha"}, + {GST_D3D11_COMPOSITOR_BLEND_DEST_ALPHA, + "The blend factor is (Ad, Ad, Ad, Ad)", "dest-alpha"}, + {GST_D3D11_COMPOSITOR_BLEND_INV_DEST_ALPHA, + "The blend factor is (1 - Ad, 1 - Ad, 1 - Ad, 1 - Ad)", + "inv-dest-alpha"}, + {GST_D3D11_COMPOSITOR_BLEND_DEST_COLOR, + "The blend factor is (Rd, Gd, Bd, Ad)", "dest-color"}, + {GST_D3D11_COMPOSITOR_BLEND_INV_DEST_COLOR, + "The blend factor is (1 - Rd, 1 - Gd, 1 - Bd, 1 - Ad)", + "inv-dest-color"}, + {GST_D3D11_COMPOSITOR_BLEND_SRC_ALPHA_SAT, + "The blend factor is (f, f, f, 1); where f = min(As, 1 - Ad)", + "src-alpha-sat"}, + {GST_D3D11_COMPOSITOR_BLEND_BLEND_FACTOR, + "User defined blend factor", "blend-factor"}, + {GST_D3D11_COMPOSITOR_BLEND_INV_BLEND_FACTOR, + "Inverse of user defined blend factor", "inv-blend-factor"}, + {0, NULL, NULL}, + }; + + if (!blend_type) { + blend_type = g_enum_register_static ("GstD3D11CompositorBlend", blend); + } + return blend_type; +} + +static GstD3D11CompositorBlend +gst_d3d11_compositor_blend_from_native (D3D11_BLEND blend) +{ + switch (blend) { + case D3D11_BLEND_ZERO: + return GST_D3D11_COMPOSITOR_BLEND_ZERO; + case D3D11_BLEND_ONE: + return GST_D3D11_COMPOSITOR_BLEND_ONE; + case D3D11_BLEND_SRC_COLOR: + return GST_D3D11_COMPOSITOR_BLEND_SRC_COLOR; + case D3D11_BLEND_INV_SRC_COLOR: + return GST_D3D11_COMPOSITOR_BLEND_INV_SRC_COLOR; + case D3D11_BLEND_SRC_ALPHA: + return GST_D3D11_COMPOSITOR_BLEND_SRC_ALPHA; + case D3D11_BLEND_INV_SRC_ALPHA: + return GST_D3D11_COMPOSITOR_BLEND_INV_SRC_ALPHA; + case D3D11_BLEND_DEST_ALPHA: + return GST_D3D11_COMPOSITOR_BLEND_DEST_ALPHA; + case D3D11_BLEND_INV_DEST_ALPHA: + return GST_D3D11_COMPOSITOR_BLEND_INV_DEST_ALPHA; + case D3D11_BLEND_DEST_COLOR: + return GST_D3D11_COMPOSITOR_BLEND_DEST_COLOR; + case D3D11_BLEND_INV_DEST_COLOR: + return GST_D3D11_COMPOSITOR_BLEND_INV_DEST_COLOR; + case D3D11_BLEND_SRC_ALPHA_SAT: + return GST_D3D11_COMPOSITOR_BLEND_SRC_ALPHA_SAT; + case D3D11_BLEND_BLEND_FACTOR: + return GST_D3D11_COMPOSITOR_BLEND_BLEND_FACTOR; + case D3D11_BLEND_INV_BLEND_FACTOR: + return GST_D3D11_COMPOSITOR_BLEND_INV_BLEND_FACTOR; + default: + g_assert_not_reached (); + break; + } + + return GST_D3D11_COMPOSITOR_BLEND_ZERO; +} + +static D3D11_BLEND +gst_d3d11_compositor_blend_to_native (GstD3D11CompositorBlend blend) +{ + switch (blend) { + case GST_D3D11_COMPOSITOR_BLEND_ZERO: + return D3D11_BLEND_ZERO; + case GST_D3D11_COMPOSITOR_BLEND_ONE: + return D3D11_BLEND_ONE; + case GST_D3D11_COMPOSITOR_BLEND_SRC_COLOR: + return D3D11_BLEND_SRC_COLOR; + case GST_D3D11_COMPOSITOR_BLEND_INV_SRC_COLOR: + return D3D11_BLEND_INV_SRC_COLOR; + case GST_D3D11_COMPOSITOR_BLEND_SRC_ALPHA: + return D3D11_BLEND_SRC_ALPHA; + case GST_D3D11_COMPOSITOR_BLEND_INV_SRC_ALPHA: + return D3D11_BLEND_INV_SRC_ALPHA; + case GST_D3D11_COMPOSITOR_BLEND_DEST_ALPHA: + return D3D11_BLEND_DEST_ALPHA; + case GST_D3D11_COMPOSITOR_BLEND_INV_DEST_ALPHA: + return D3D11_BLEND_INV_DEST_ALPHA; + case GST_D3D11_COMPOSITOR_BLEND_DEST_COLOR: + return D3D11_BLEND_DEST_COLOR; + case GST_D3D11_COMPOSITOR_BLEND_INV_DEST_COLOR: + return D3D11_BLEND_INV_DEST_COLOR; + case GST_D3D11_COMPOSITOR_BLEND_SRC_ALPHA_SAT: + return D3D11_BLEND_SRC_ALPHA_SAT; + case GST_D3D11_COMPOSITOR_BLEND_BLEND_FACTOR: + return D3D11_BLEND_BLEND_FACTOR; + case GST_D3D11_COMPOSITOR_BLEND_INV_BLEND_FACTOR: + return D3D11_BLEND_INV_BLEND_FACTOR; + default: + g_assert_not_reached (); + break; + } + + return D3D11_BLEND_ZERO; +} + +GType +gst_d3d11_compositor_background_get_type (void) +{ + static GType compositor_background_type = 0; + + static const GEnumValue compositor_background[] = { + {GST_D3D11_COMPOSITOR_BACKGROUND_CHECKER, "Checker pattern", "checker"}, + {GST_D3D11_COMPOSITOR_BACKGROUND_BLACK, "Black", "black"}, + {GST_D3D11_COMPOSITOR_BACKGROUND_WHITE, "White", "white"}, + {GST_D3D11_COMPOSITOR_BACKGROUND_TRANSPARENT, + "Transparent Background to enable further compositing", "transparent"}, + {0, NULL, NULL}, + }; + + if (!compositor_background_type) { + compositor_background_type = + g_enum_register_static ("GstD3D11CompositorBackground", + compositor_background); + } + return compositor_background_type; +} + +/* *INDENT-OFF* */ +static const gchar checker_vs_src[] = + "struct VS_INPUT\n" + "{\n" + " float4 Position : POSITION;\n" + "};\n" + "\n" + "struct VS_OUTPUT\n" + "{\n" + " float4 Position: SV_POSITION;\n" + "};\n" + "\n" + "VS_OUTPUT main(VS_INPUT input)\n" + "{\n" + " return input;\n" + "}\n"; + +static const gchar checker_ps_src[] = + "static const float blocksize = 8.0;\n" + "static const float4 high = float4(0.667, 0.667, 0.667, 1.0);\n" + "static const float4 low = float4(0.333, 0.333, 0.333, 1.0);\n" + "struct PS_INPUT\n" + "{\n" + " float4 Position: SV_POSITION;\n" + "};\n" + "struct PS_OUTPUT\n" + "{\n" + " float4 Plane: SV_TARGET;\n" + "};\n" + "PS_OUTPUT main(PS_INPUT input)\n" + "{\n" + " PS_OUTPUT output;\n" + " if ((input.Position.x % (blocksize * 2.0)) >= blocksize) {\n" + " if ((input.Position.y % (blocksize * 2.0)) >= blocksize)\n" + " output.Plane = low;\n" + " else\n" + " output.Plane = high;\n" + " } else {\n" + " if ((input.Position.y % (blocksize * 2.0)) < blocksize)\n" + " output.Plane = low;\n" + " else\n" + " output.Plane = high;\n" + " }\n" + " return output;\n" + "}\n"; +/* *INDENT-ON* */ + +struct _GstD3D11CompositorPad +{ + GstVideoAggregatorConvertPad parent; + + GstD3D11ColorConverter *convert; + + GstBufferPool *fallback_pool; + GstBuffer *fallback_buf; + + gboolean position_updated; + gboolean alpha_updated; + gboolean blend_desc_updated; + ID3D11BlendState *blend; + + /* properties */ + gint xpos; + gint ypos; + gint width; + gint height; + gdouble alpha; + D3D11_RENDER_TARGET_BLEND_DESC desc; + gfloat blend_factor[4]; +}; + +struct _GstD3D11Compositor +{ + GstVideoAggregator parent; + + GstD3D11Device *device; + + GstBufferPool *fallback_pool; + GstBuffer *fallback_buf; + + GstD3D11Quad *checker_background; + D3D11_VIEWPORT viewport; + + /* properties */ + gint adapter; + GstD3D11CompositorBackground background; +}; + +enum +{ + PROP_PAD_0, + PROP_PAD_XPOS, + PROP_PAD_YPOS, + PROP_PAD_WIDTH, + PROP_PAD_HEIGHT, + PROP_PAD_ALPHA, + PROP_PAD_BLEND_OP_RGB, + PROP_PAD_BLEND_OP_ALPHA, + PROP_PAD_BLEND_SRC_RGB, + PROP_PAD_BLEND_SRC_ALPHA, + PROP_PAD_BLEND_DEST_RGB, + PROP_PAD_BLEND_DEST_ALPHA, + PROP_PAD_BLEND_FACTOR_RED, + PROP_PAD_BLEND_FACTOR_GREEN, + PROP_PAD_BLEND_FACTOR_BLUE, + PROP_PAD_BLEND_FACTOR_ALPHA, +}; + +#define DEFAULT_PAD_XPOS 0 +#define DEFAULT_PAD_YPOS 0 +#define DEFAULT_PAD_WIDTH 0 +#define DEFAULT_PAD_HEIGHT 0 +#define DEFAULT_PAD_ALPHA 1.0 +#define DEFAULT_PAD_BLEND_OP_RGB GST_D3D11_COMPOSITOR_BLEND_OP_ADD +#define DEFAULT_PAD_BLEND_OP_ALPHA GST_D3D11_COMPOSITOR_BLEND_OP_ADD +#define DEFAULT_PAD_BLEND_SRC_RGB GST_D3D11_COMPOSITOR_BLEND_SRC_ALPHA +#define DEFAULT_PAD_BLEND_SRC_ALPHA GST_D3D11_COMPOSITOR_BLEND_ONE +#define DEFAULT_PAD_BLEND_DEST_RGB GST_D3D11_COMPOSITOR_BLEND_INV_SRC_ALPHA +#define DEFAULT_PAD_BLEND_DEST_ALPHA GST_D3D11_COMPOSITOR_BLEND_INV_SRC_ALPHA + +static void gst_d3d11_compositor_pad_set_property (GObject * object, + guint prop_id, const GValue * value, GParamSpec * pspec); +static void gst_d3d11_compositor_pad_get_property (GObject * object, + guint prop_id, GValue * value, GParamSpec * pspec); +static gboolean +gst_d3d11_compositor_pad_prepare_frame (GstVideoAggregatorPad * pad, + GstVideoAggregator * vagg, GstBuffer * buffer, + GstVideoFrame * prepared_frame); +static void +gst_d3d11_compositor_pad_clean_frame (GstVideoAggregatorPad * pad, + GstVideoAggregator * vagg, GstVideoFrame * prepared_frame); +static void +gst_d3d11_compositor_pad_init_blend_options (GstD3D11CompositorPad * pad); + +#define gst_d3d11_compositor_pad_parent_class parent_pad_class +G_DEFINE_TYPE (GstD3D11CompositorPad, gst_d3d11_compositor_pad, + GST_TYPE_VIDEO_AGGREGATOR_PAD); + +static void +gst_d3d11_compositor_pad_class_init (GstD3D11CompositorPadClass * klass) +{ + GObjectClass *gobject_class = G_OBJECT_CLASS (klass); + GstVideoAggregatorPadClass *vaggpadclass = + GST_VIDEO_AGGREGATOR_PAD_CLASS (klass); + + gobject_class->set_property = gst_d3d11_compositor_pad_set_property; + gobject_class->get_property = gst_d3d11_compositor_pad_get_property; + + g_object_class_install_property (gobject_class, PROP_PAD_XPOS, + g_param_spec_int ("xpos", "X Position", "X position of the picture", + G_MININT, G_MAXINT, DEFAULT_PAD_XPOS, + G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); + + g_object_class_install_property (gobject_class, PROP_PAD_YPOS, + g_param_spec_int ("ypos", "Y Position", "Y position of the picture", + G_MININT, G_MAXINT, DEFAULT_PAD_YPOS, + G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); + + g_object_class_install_property (gobject_class, PROP_PAD_WIDTH, + g_param_spec_int ("width", "Width", "Width of the picture", + G_MININT, G_MAXINT, DEFAULT_PAD_WIDTH, + G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); + + g_object_class_install_property (gobject_class, PROP_PAD_HEIGHT, + g_param_spec_int ("height", "Height", "Height of the picture", + G_MININT, G_MAXINT, DEFAULT_PAD_HEIGHT, + G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); + + g_object_class_install_property (gobject_class, PROP_PAD_ALPHA, + g_param_spec_double ("alpha", "Alpha", "Alpha of the picture", 0.0, 1.0, + DEFAULT_PAD_ALPHA, + G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); + + g_object_class_install_property (gobject_class, PROP_PAD_BLEND_OP_RGB, + g_param_spec_enum ("blend-op-rgb", "Blend Operation RGB", + "Blend equation for RGB", GST_TYPE_D3D11_COMPOSITOR_BLEND_OPERATION, + DEFAULT_PAD_BLEND_OP_RGB, + G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); + + g_object_class_install_property (gobject_class, PROP_PAD_BLEND_OP_ALPHA, + g_param_spec_enum ("blend-op-alpha", "Blend Operation Alpha", + "Blend equation for alpha", GST_TYPE_D3D11_COMPOSITOR_BLEND_OPERATION, + DEFAULT_PAD_BLEND_OP_ALPHA, + G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); + + g_object_class_install_property (gobject_class, + PROP_PAD_BLEND_SRC_RGB, + g_param_spec_enum ("blend-src-rgb", "Blend Source RGB", + "Blend factor for source RGB", + GST_TYPE_D3D11_COMPOSITOR_BLEND, + DEFAULT_PAD_BLEND_SRC_RGB, + G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); + + g_object_class_install_property (gobject_class, + PROP_PAD_BLEND_SRC_ALPHA, + g_param_spec_enum ("blend-src-alpha", + "Blend Source Alpha", + "Blend factor for source alpha, \"*-color\" values are not allowed", + GST_TYPE_D3D11_COMPOSITOR_BLEND, + DEFAULT_PAD_BLEND_SRC_ALPHA, + G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); + + g_object_class_install_property (gobject_class, + PROP_PAD_BLEND_DEST_RGB, + g_param_spec_enum ("blend-dest-rgb", + "Blend Destination RGB", + "Blend factor for destination RGB", + GST_TYPE_D3D11_COMPOSITOR_BLEND, + DEFAULT_PAD_BLEND_DEST_RGB, + G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); + + g_object_class_install_property (gobject_class, + PROP_PAD_BLEND_DEST_ALPHA, + g_param_spec_enum ("blend-dest-alpha", + "Blend Destination Alpha", + "Blend factor for destination alpha, " + "\"*-color\" values are not allowed", + GST_TYPE_D3D11_COMPOSITOR_BLEND, + DEFAULT_PAD_BLEND_DEST_ALPHA, + G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); + + g_object_class_install_property (gobject_class, PROP_PAD_BLEND_FACTOR_RED, + g_param_spec_float ("blend-factor-red", "Blend Factor Red", + "Blend factor for red component " + "when blend type is \"blend-factor\" or \"inv-blend-factor\"", + 0.0, 1.0, 1.0, + G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); + + g_object_class_install_property (gobject_class, PROP_PAD_BLEND_FACTOR_GREEN, + g_param_spec_float ("blend-factor-green", "Blend Factor Green", + "Blend factor for green component " + "when blend type is \"blend-factor\" or \"inv-blend-factor\"", + 0.0, 1.0, 1.0, + G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); + + g_object_class_install_property (gobject_class, PROP_PAD_BLEND_FACTOR_BLUE, + g_param_spec_float ("blend-factor-blue", "Blend Factor Blue", + "Blend factor for blue component " + "when blend type is \"blend-factor\" or \"inv-blend-factor\"", + 0.0, 1.0, 1.0, + G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); + + g_object_class_install_property (gobject_class, PROP_PAD_BLEND_FACTOR_ALPHA, + g_param_spec_float ("blend-factor-alpha", "Blend Factor Alpha", + "Blend factor for alpha component " + "when blend type is \"blend-factor\" or \"inv-blend-factor\"", + 0.0, 1.0, 1.0, + G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); + + vaggpadclass->prepare_frame = + GST_DEBUG_FUNCPTR (gst_d3d11_compositor_pad_prepare_frame); + vaggpadclass->clean_frame = + GST_DEBUG_FUNCPTR (gst_d3d11_compositor_pad_clean_frame); +} + +static void +gst_d3d11_compositor_pad_init (GstD3D11CompositorPad * pad) +{ + pad->xpos = DEFAULT_PAD_XPOS; + pad->ypos = DEFAULT_PAD_YPOS; + pad->width = DEFAULT_PAD_WIDTH; + pad->height = DEFAULT_PAD_HEIGHT; + pad->alpha = DEFAULT_PAD_ALPHA; + + gst_d3d11_compositor_pad_init_blend_options (pad); +} + +static void +gst_d3d11_compositor_pad_update_blend_function (GstD3D11CompositorPad * pad, + D3D11_BLEND * value, GstD3D11CompositorBlend new_value) +{ + D3D11_BLEND temp = gst_d3d11_compositor_blend_to_native (new_value); + + if (temp == *value) + return; + + *value = temp; + pad->blend_desc_updated = TRUE; +} + +static void +gst_d3d11_compositor_pad_update_blend_equation (GstD3D11CompositorPad * pad, + D3D11_BLEND_OP * value, GstD3D11CompositorBlend new_value) +{ + D3D11_BLEND_OP temp = + gst_d3d11_compositor_blend_operation_to_native (new_value); + + if (temp == *value) + return; + + *value = temp; + pad->blend_desc_updated = TRUE; +} + +static void +gst_d3d11_compositor_pad_set_property (GObject * object, guint prop_id, + const GValue * value, GParamSpec * pspec) +{ + GstD3D11CompositorPad *pad = GST_D3D11_COMPOSITOR_PAD (object); + + switch (prop_id) { + case PROP_PAD_XPOS: + pad->xpos = g_value_get_int (value); + pad->position_updated = TRUE; + break; + case PROP_PAD_YPOS: + pad->ypos = g_value_get_int (value); + pad->position_updated = TRUE; + break; + case PROP_PAD_WIDTH: + pad->width = g_value_get_int (value); + pad->position_updated = TRUE; + break; + case PROP_PAD_HEIGHT: + pad->height = g_value_get_int (value); + pad->position_updated = TRUE; + break; + case PROP_PAD_ALPHA: + { + gdouble alpha = g_value_get_double (value); + if (pad->alpha != alpha) { + pad->alpha_updated = TRUE; + pad->alpha = alpha; + } + break; + } + case PROP_PAD_BLEND_OP_RGB: + gst_d3d11_compositor_pad_update_blend_equation (pad, &pad->desc.BlendOp, + g_value_get_enum (value)); + break; + case PROP_PAD_BLEND_OP_ALPHA: + gst_d3d11_compositor_pad_update_blend_equation (pad, + &pad->desc.BlendOpAlpha, g_value_get_enum (value)); + break; + case PROP_PAD_BLEND_SRC_RGB: + gst_d3d11_compositor_pad_update_blend_function (pad, &pad->desc.SrcBlend, + g_value_get_enum (value)); + break; + case PROP_PAD_BLEND_SRC_ALPHA: + { + GstD3D11CompositorBlend blend = g_value_get_enum (value); + if (blend == GST_D3D11_COMPOSITOR_BLEND_SRC_COLOR || + blend == GST_D3D11_COMPOSITOR_BLEND_INV_SRC_COLOR || + blend == GST_D3D11_COMPOSITOR_BLEND_DEST_COLOR || + blend == GST_D3D11_COMPOSITOR_BLEND_INV_DEST_COLOR) { + g_warning ("%d is not allowed for %s", blend, pspec->name); + } else { + gst_d3d11_compositor_pad_update_blend_function (pad, + &pad->desc.SrcBlendAlpha, blend); + } + break; + } + case PROP_PAD_BLEND_DEST_RGB: + gst_d3d11_compositor_pad_update_blend_function (pad, &pad->desc.DestBlend, + g_value_get_enum (value)); + break; + case PROP_PAD_BLEND_DEST_ALPHA: + { + GstD3D11CompositorBlend blend = g_value_get_enum (value); + if (blend == GST_D3D11_COMPOSITOR_BLEND_SRC_COLOR || + blend == GST_D3D11_COMPOSITOR_BLEND_INV_SRC_COLOR || + blend == GST_D3D11_COMPOSITOR_BLEND_DEST_COLOR || + blend == GST_D3D11_COMPOSITOR_BLEND_INV_DEST_COLOR) { + g_warning ("%d is not allowed for %s", blend, pspec->name); + } else { + gst_d3d11_compositor_pad_update_blend_function (pad, + &pad->desc.DestBlendAlpha, blend); + } + break; + } + case PROP_PAD_BLEND_FACTOR_RED: + pad->blend_factor[0] = g_value_get_float (value); + break; + case PROP_PAD_BLEND_FACTOR_GREEN: + pad->blend_factor[1] = g_value_get_float (value); + break; + case PROP_PAD_BLEND_FACTOR_BLUE: + pad->blend_factor[2] = g_value_get_float (value); + break; + case PROP_PAD_BLEND_FACTOR_ALPHA: + pad->blend_factor[3] = g_value_get_float (value); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static void +gst_d3d11_compositor_pad_get_property (GObject * object, guint prop_id, + GValue * value, GParamSpec * pspec) +{ + GstD3D11CompositorPad *pad = GST_D3D11_COMPOSITOR_PAD (object); + + switch (prop_id) { + case PROP_PAD_XPOS: + g_value_set_int (value, pad->xpos); + break; + case PROP_PAD_YPOS: + g_value_set_int (value, pad->ypos); + break; + case PROP_PAD_WIDTH: + g_value_set_int (value, pad->width); + break; + case PROP_PAD_HEIGHT: + g_value_set_int (value, pad->height); + break; + case PROP_PAD_ALPHA: + g_value_set_double (value, pad->alpha); + break; + case PROP_PAD_BLEND_OP_RGB: + g_value_set_enum (value, + gst_d3d11_compositor_blend_operation_from_native (pad->desc.BlendOp)); + break; + case PROP_PAD_BLEND_OP_ALPHA: + g_value_set_enum (value, + gst_d3d11_compositor_blend_operation_from_native (pad-> + desc.BlendOpAlpha)); + break; + case PROP_PAD_BLEND_SRC_RGB: + g_value_set_enum (value, + gst_d3d11_compositor_blend_from_native (pad->desc.SrcBlend)); + break; + case PROP_PAD_BLEND_SRC_ALPHA: + g_value_set_enum (value, + gst_d3d11_compositor_blend_from_native (pad->desc.SrcBlendAlpha)); + break; + case PROP_PAD_BLEND_DEST_RGB: + g_value_set_enum (value, + gst_d3d11_compositor_blend_from_native (pad->desc.DestBlend)); + break; + case PROP_PAD_BLEND_DEST_ALPHA: + g_value_set_enum (value, + gst_d3d11_compositor_blend_from_native (pad->desc.DestBlendAlpha)); + break; + case PROP_PAD_BLEND_FACTOR_RED: + g_value_set_float (value, pad->blend_factor[0]); + break; + case PROP_PAD_BLEND_FACTOR_GREEN: + g_value_set_float (value, pad->blend_factor[1]); + break; + case PROP_PAD_BLEND_FACTOR_BLUE: + g_value_set_float (value, pad->blend_factor[2]); + break; + case PROP_PAD_BLEND_FACTOR_ALPHA: + g_value_set_float (value, pad->blend_factor[3]); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static void +gst_d3d11_compositor_pad_init_blend_options (GstD3D11CompositorPad * pad) +{ + gint i; + + pad->desc.BlendEnable = TRUE; + pad->desc.SrcBlend = + gst_d3d11_compositor_blend_to_native (DEFAULT_PAD_BLEND_SRC_RGB); + pad->desc.DestBlend = + gst_d3d11_compositor_blend_to_native (DEFAULT_PAD_BLEND_DEST_RGB); + pad->desc.BlendOp = + gst_d3d11_compositor_blend_operation_to_native (DEFAULT_PAD_BLEND_OP_RGB); + pad->desc.SrcBlendAlpha = + gst_d3d11_compositor_blend_to_native (DEFAULT_PAD_BLEND_SRC_ALPHA); + pad->desc.DestBlendAlpha = + gst_d3d11_compositor_blend_to_native (DEFAULT_PAD_BLEND_DEST_ALPHA); + pad->desc.BlendOpAlpha = + gst_d3d11_compositor_blend_operation_to_native + (DEFAULT_PAD_BLEND_OP_ALPHA); + pad->desc.RenderTargetWriteMask = D3D11_COLOR_WRITE_ENABLE_ALL; + + for (i = 0; i < G_N_ELEMENTS (pad->blend_factor); i++) + pad->blend_factor[i] = 1.0f; +} + +static gboolean +gst_d3d11_compositor_configure_fallback_pool (GstD3D11Compositor * self, + GstVideoInfo * info, gint bind_flags, GstBufferPool ** pool) +{ + GstStructure *config; + GstD3D11AllocationParams *d3d11_params; + GstCaps *caps; + + if (*pool) { + gst_buffer_pool_set_active (*pool, FALSE); + gst_clear_object (pool); + } + + caps = gst_video_info_to_caps (info); + if (!caps) { + GST_ERROR_OBJECT (self, "Couldn't create caps from info"); + return FALSE; + } + + *pool = gst_d3d11_buffer_pool_new (self->device); + config = gst_buffer_pool_get_config (*pool); + gst_buffer_pool_config_set_params (config, + caps, GST_VIDEO_INFO_SIZE (info), 0, 0); + gst_caps_unref (caps); + + d3d11_params = gst_buffer_pool_config_get_d3d11_allocation_params (config); + if (!d3d11_params) { + d3d11_params = gst_d3d11_allocation_params_new (self->device, + info, 0, bind_flags); + } else { + gint i; + for (i = 0; i < GST_VIDEO_INFO_N_PLANES (info); i++) + d3d11_params->desc[i].BindFlags |= bind_flags; + } + + gst_buffer_pool_config_set_d3d11_allocation_params (config, d3d11_params); + gst_d3d11_allocation_params_free (d3d11_params); + + gst_buffer_pool_config_add_option (config, GST_BUFFER_POOL_OPTION_VIDEO_META); + gst_buffer_pool_set_config (*pool, config); + gst_buffer_pool_set_active (*pool, TRUE); + + return TRUE; +} + +static gboolean +gst_d3d11_compsitor_prepare_fallback_buffer (GstD3D11Compositor * self, + GstVideoInfo * info, gboolean is_input, GstBufferPool ** pool, + GstBuffer ** fallback_buffer) +{ + GstBuffer *new_buf = NULL; + gint bind_flags = D3D11_BIND_SHADER_RESOURCE; + gint i; + + gst_clear_buffer (fallback_buffer); + + if (!is_input) + bind_flags = D3D11_BIND_RENDER_TARGET; + + if (*pool == NULL && + !gst_d3d11_compositor_configure_fallback_pool (self, info, + bind_flags, pool)) { + GST_ERROR_OBJECT (self, "Couldn't configure fallback buffer pool"); + return FALSE; + } + + if (gst_buffer_pool_acquire_buffer (*pool, &new_buf, NULL) + != GST_FLOW_OK) { + GST_ERROR_OBJECT (self, "Couldn't get fallback buffer from pool"); + return FALSE; + } + + for (i = 0; i < gst_buffer_n_memory (new_buf); i++) { + GstD3D11Memory *new_mem = + (GstD3D11Memory *) gst_buffer_peek_memory (new_buf, i); + + if (is_input && !gst_d3d11_memory_ensure_shader_resource_view (new_mem)) { + GST_ERROR_OBJECT (self, "Couldn't prepare shader resource view"); + gst_buffer_unref (new_buf); + return FALSE; + } else if (!is_input && + !gst_d3d11_memory_ensure_render_target_view (new_mem)) { + GST_ERROR_OBJECT (self, "Couldn't prepare render target view"); + gst_buffer_unref (new_buf); + return FALSE; + } + } + + *fallback_buffer = new_buf; + + return TRUE; +} + +static gboolean +gst_d3d11_compositor_copy_buffer (GstD3D11Compositor * self, + GstVideoInfo * info, GstBuffer * src_buf, GstBuffer * dest_buf, + gboolean do_device_copy) +{ + gint i; + + if (do_device_copy) { + return gst_d3d11_buffer_copy_into (dest_buf, src_buf); + } else { + GstVideoFrame src_frame, dest_frame; + + if (!gst_video_frame_map (&src_frame, info, src_buf, + GST_MAP_READ | GST_VIDEO_FRAME_MAP_FLAG_NO_REF)) { + GST_ERROR_OBJECT (self, "Couldn't map input buffer"); + return FALSE; + } + + if (!gst_video_frame_map (&dest_frame, info, dest_buf, + GST_MAP_WRITE | GST_VIDEO_FRAME_MAP_FLAG_NO_REF)) { + GST_ERROR_OBJECT (self, "Couldn't fallback buffer"); + gst_video_frame_unmap (&src_frame); + return FALSE; + } + + for (i = 0; i < GST_VIDEO_FRAME_N_PLANES (&src_frame); i++) { + if (!gst_video_frame_copy_plane (&dest_frame, &src_frame, i)) { + GST_ERROR_OBJECT (self, "Couldn't copy %dth plane", i); + + gst_video_frame_unmap (&dest_frame); + gst_video_frame_unmap (&src_frame); + + return FALSE; + } + } + + gst_video_frame_unmap (&dest_frame); + gst_video_frame_unmap (&src_frame); + } + + return TRUE; +} + +static gboolean +gst_d3d11_compositor_check_d3d11_memory (GstD3D11Compositor * self, + GstBuffer * buffer, gboolean is_input, gboolean * view_available) +{ + gint i; + gboolean ret = TRUE; + + *view_available = TRUE; + + for (i = 0; i < gst_buffer_n_memory (buffer); i++) { + GstMemory *mem = gst_buffer_peek_memory (buffer, i); + GstD3D11Memory *dmem; + + if (!gst_is_d3d11_memory (mem)) { + ret = FALSE; + goto done; + } + + dmem = (GstD3D11Memory *) mem; + if (dmem->device != self->device) { + ret = FALSE; + goto done; + } + + if (is_input) { + if (!gst_d3d11_memory_ensure_shader_resource_view (dmem)) + *view_available = FALSE; + } else { + if (!gst_d3d11_memory_ensure_render_target_view (dmem)) + *view_available = FALSE; + } + } + +done: + if (!ret) + *view_available = FALSE; + + return ret; +} + +static void +gst_d3d11_compositor_pad_get_output_size (GstD3D11CompositorPad * comp_pad, + gint out_par_n, gint out_par_d, gint * width, gint * height) +{ + GstVideoAggregatorPad *vagg_pad = GST_VIDEO_AGGREGATOR_PAD (comp_pad); + gint pad_width, pad_height; + guint dar_n, dar_d; + + /* FIXME: Anything better we can do here? */ + if (!vagg_pad->info.finfo + || vagg_pad->info.finfo->format == GST_VIDEO_FORMAT_UNKNOWN) { + GST_DEBUG_OBJECT (comp_pad, "Have no caps yet"); + *width = 0; + *height = 0; + return; + } + + pad_width = + comp_pad->width <= + 0 ? GST_VIDEO_INFO_WIDTH (&vagg_pad->info) : comp_pad->width; + pad_height = + comp_pad->height <= + 0 ? GST_VIDEO_INFO_HEIGHT (&vagg_pad->info) : comp_pad->height; + + if (!gst_video_calculate_display_ratio (&dar_n, &dar_d, pad_width, pad_height, + GST_VIDEO_INFO_PAR_N (&vagg_pad->info), + GST_VIDEO_INFO_PAR_D (&vagg_pad->info), out_par_n, out_par_d)) { + GST_WARNING_OBJECT (comp_pad, "Cannot calculate display aspect ratio"); + *width = *height = 0; + return; + } + + GST_TRACE_OBJECT (comp_pad, "scaling %ux%u by %u/%u (%u/%u / %u/%u)", + pad_width, pad_height, dar_n, dar_d, + GST_VIDEO_INFO_PAR_N (&vagg_pad->info), + GST_VIDEO_INFO_PAR_D (&vagg_pad->info), out_par_n, out_par_d); + + /* Pick either height or width, whichever is an integer multiple of the + * display aspect ratio. However, prefer preserving the height to account + * for interlaced video. */ + if (pad_height % dar_n == 0) { + pad_width = gst_util_uint64_scale_int (pad_height, dar_n, dar_d); + } else if (pad_width % dar_d == 0) { + pad_height = gst_util_uint64_scale_int (pad_width, dar_d, dar_n); + } else { + pad_width = gst_util_uint64_scale_int (pad_height, dar_n, dar_d); + } + + *width = pad_width; + *height = pad_height; +} + +static GstVideoRectangle +clamp_rectangle (gint x, gint y, gint w, gint h, gint outer_width, + gint outer_height) +{ + gint x2 = x + w; + gint y2 = y + h; + GstVideoRectangle clamped; + + /* Clamp the x/y coordinates of this frame to the output boundaries to cover + * the case where (say, with negative xpos/ypos or w/h greater than the output + * size) the non-obscured portion of the frame could be outside the bounds of + * the video itself and hence not visible at all */ + clamped.x = CLAMP (x, 0, outer_width); + clamped.y = CLAMP (y, 0, outer_height); + clamped.w = CLAMP (x2, 0, outer_width) - clamped.x; + clamped.h = CLAMP (y2, 0, outer_height) - clamped.y; + + return clamped; +} + +static gboolean +gst_d3d11_compositor_pad_check_frame_obscured (GstVideoAggregatorPad * pad, + GstVideoAggregator * vagg) +{ + GstD3D11CompositorPad *cpad = GST_D3D11_COMPOSITOR_PAD (pad); + gint width, height; + GstVideoInfo *info = &vagg->info; + /* The rectangle representing this frame, clamped to the video's boundaries. + * Due to the clamping, this is different from the frame width/height above. */ + GstVideoRectangle frame_rect; + + /* There's three types of width/height here: + * 1. GST_VIDEO_FRAME_WIDTH/HEIGHT: + * The frame width/height (same as pad->info.height/width; + * see gst_video_frame_map()) + * 2. cpad->width/height: + * The optional pad property for scaling the frame (if zero, the video is + * left unscaled) + */ + + gst_d3d11_compositor_pad_get_output_size (cpad, GST_VIDEO_INFO_PAR_N (info), + GST_VIDEO_INFO_PAR_D (info), &width, &height); + + frame_rect = clamp_rectangle (cpad->xpos, cpad->ypos, width, height, + GST_VIDEO_INFO_WIDTH (info), GST_VIDEO_INFO_HEIGHT (info)); + + if (frame_rect.w == 0 || frame_rect.h == 0) { + GST_DEBUG_OBJECT (pad, "Resulting frame is zero-width or zero-height " + "(w: %i, h: %i), skipping", frame_rect.w, frame_rect.h); + return TRUE; + } + + return FALSE; +} + +static gboolean +gst_d3d11_compositor_pad_prepare_frame (GstVideoAggregatorPad * pad, + GstVideoAggregator * vagg, GstBuffer * buffer, + GstVideoFrame * prepared_frame) +{ + GstD3D11Compositor *self = GST_D3D11_COMPOSITOR (vagg); + GstD3D11CompositorPad *cpad = GST_D3D11_COMPOSITOR_PAD (pad); + GstBuffer *target_buf = buffer; + gboolean do_device_copy = FALSE; + + /* Skip this frame */ + if (gst_d3d11_compositor_pad_check_frame_obscured (pad, vagg)) + return TRUE; + + /* Use fallback buffer when input buffer is: + * - non-d3d11 memory + * - or, from different d3d11 device + * - or not bound to shader resource + */ + if (!gst_d3d11_compositor_check_d3d11_memory (self, + buffer, TRUE, &do_device_copy) || !do_device_copy) { + if (!gst_d3d11_compsitor_prepare_fallback_buffer (self, &pad->info, TRUE, + &cpad->fallback_pool, &cpad->fallback_buf)) { + GST_ERROR_OBJECT (self, "Couldn't prepare fallback buffer"); + return FALSE; + } + + if (!gst_d3d11_compositor_copy_buffer (self, &pad->info, buffer, + cpad->fallback_buf, do_device_copy)) { + GST_ERROR_OBJECT (self, "Couldn't copy input buffer to fallback buffer"); + gst_clear_buffer (&cpad->fallback_buf); + return FALSE; + } + + target_buf = cpad->fallback_buf; + } + + if (!gst_video_frame_map (prepared_frame, &pad->info, target_buf, + GST_MAP_READ | GST_MAP_D3D11)) { + GST_WARNING_OBJECT (pad, "Couldn't map input buffer"); + return FALSE; + } + + return TRUE; +} + +static void +gst_d3d11_compositor_pad_clean_frame (GstVideoAggregatorPad * pad, + GstVideoAggregator * vagg, GstVideoFrame * prepared_frame) +{ + GstD3D11CompositorPad *cpad = GST_D3D11_COMPOSITOR_PAD (pad); + + GST_VIDEO_AGGREGATOR_PAD_CLASS (parent_pad_class)->clean_frame (pad, + vagg, prepared_frame); + + gst_clear_buffer (&cpad->fallback_buf); +} + +static gboolean +gst_d3d11_compositor_pad_setup_converter (GstVideoAggregatorPad * pad, + GstVideoAggregator * vagg) +{ + GstD3D11CompositorPad *cpad = GST_D3D11_COMPOSITOR_PAD (pad); + GstD3D11Compositor *self = GST_D3D11_COMPOSITOR (vagg); + RECT rect; + gint width, height; + GstVideoInfo *info = &vagg->info; + GstVideoRectangle frame_rect; + gboolean is_first = FALSE; +#ifndef GST_DISABLE_GST_DEBUG + guint zorder = 0; +#endif + + if (!cpad->convert || cpad->alpha_updated) { + if (cpad->convert) + gst_d3d11_color_converter_free (cpad->convert); + cpad->convert = + gst_d3d11_color_converter_new_with_alpha (self->device, + &pad->info, &vagg->info, cpad->alpha); + cpad->alpha_updated = FALSE; + if (!cpad->convert) { + GST_ERROR_OBJECT (pad, "Couldn't create converter"); + return FALSE; + } + + is_first = TRUE; + } + + if (!cpad->blend || cpad->blend_desc_updated) { + HRESULT hr; + D3D11_BLEND_DESC desc = { 0, }; + ID3D11BlendState *blend = NULL; + ID3D11Device *device_handle = + gst_d3d11_device_get_device_handle (self->device); + + if (cpad->blend) { + ID3D11BlendState_Release (cpad->blend); + cpad->blend = NULL; + } + + desc.AlphaToCoverageEnable = FALSE; + desc.IndependentBlendEnable = FALSE; + desc.RenderTarget[0] = cpad->desc; + + hr = ID3D11Device_CreateBlendState (device_handle, &desc, &blend); + if (!gst_d3d11_result (hr, self->device)) { + GST_ERROR_OBJECT (pad, "Couldn't create blend staten, hr: 0x%x", + (guint) hr); + return FALSE; + } + + cpad->blend = blend; + } + + if (!is_first && !cpad->position_updated) + return TRUE; + + gst_d3d11_compositor_pad_get_output_size (cpad, GST_VIDEO_INFO_PAR_N (info), + GST_VIDEO_INFO_PAR_D (info), &width, &height); + + frame_rect = clamp_rectangle (cpad->xpos, cpad->ypos, width, height, + GST_VIDEO_INFO_WIDTH (info), GST_VIDEO_INFO_HEIGHT (info)); + + rect.left = frame_rect.x; + rect.top = frame_rect.y; + rect.right = frame_rect.x + frame_rect.w; + rect.bottom = frame_rect.y + frame_rect.h; + +#ifndef GST_DISABLE_GST_DEBUG + g_object_get (pad, "zorder", &zorder, NULL); + + GST_LOG_OBJECT (pad, "Update position, pad-xpos %d, pad-ypos %d, " + "pad-zorder %d, pad-width %d, pad-height %d, in-resolution %dx%d, " + "out-resoution %dx%d, dst-{left,top,right,bottom} %d-%d-%d-%d", + cpad->xpos, cpad->ypos, zorder, cpad->width, cpad->height, + GST_VIDEO_INFO_WIDTH (&pad->info), GST_VIDEO_INFO_HEIGHT (&pad->info), + GST_VIDEO_INFO_WIDTH (info), GST_VIDEO_INFO_HEIGHT (info), + (gint) rect.left, (gint) rect.top, (gint) rect.right, (gint) rect.bottom); +#endif + + cpad->position_updated = FALSE; + + return gst_d3d11_color_converter_update_dest_rect (cpad->convert, &rect); +} + +static GstStaticPadTemplate sink_template = GST_STATIC_PAD_TEMPLATE ("sink_%u", + GST_PAD_SINK, + GST_PAD_REQUEST, + GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES + (GST_CAPS_FEATURE_MEMORY_D3D11_MEMORY, "{ RGBA, BGRA }") + )); + +static GstStaticPadTemplate src_template = GST_STATIC_PAD_TEMPLATE ("src", + GST_PAD_SRC, + GST_PAD_ALWAYS, + GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES + (GST_CAPS_FEATURE_MEMORY_D3D11_MEMORY, "{ RGBA, BGRA }") + )); + +enum +{ + PROP_0, + PROP_ADAPTER, + PROP_BACKGROUND, +}; + +#define DEFAULT_ADAPTER -1 +#define DEFAULT_BACKGROUND GST_D3D11_COMPOSITOR_BACKGROUND_CHECKER + +static void gst_d3d11_compositor_child_proxy_init (gpointer g_iface, + gpointer iface_data); +static void gst_d3d11_compositor_dispose (GObject * object); +static void gst_d3d11_compositor_set_property (GObject * object, + guint prop_id, const GValue * value, GParamSpec * pspec); +static void gst_d3d11_compositor_get_property (GObject * object, + guint prop_id, GValue * value, GParamSpec * pspec); + +static GstPad *gst_d3d11_compositor_request_new_pad (GstElement * element, + GstPadTemplate * templ, const gchar * name, const GstCaps * caps); +static void gst_d3d11_compositor_release_pad (GstElement * element, + GstPad * pad); +static void gst_d3d11_compositor_set_context (GstElement * element, + GstContext * context); + +static gboolean gst_d3d11_compositor_start (GstAggregator * aggregator); +static gboolean gst_d3d11_compositor_stop (GstAggregator * aggregator); +static gboolean gst_d3d11_compositor_sink_query (GstAggregator * aggregator, + GstAggregatorPad * pad, GstQuery * query); +static gboolean gst_d3d11_compositor_src_query (GstAggregator * aggregator, + GstQuery * query); +static GstCaps *gst_d3d11_compositor_fixate_src_caps (GstAggregator * + aggregator, GstCaps * caps); +static gboolean gst_d3d11_compositor_propose_allocation (GstAggregator * + aggregator, GstAggregatorPad * pad, GstQuery * decide_query, + GstQuery * query); +static gboolean gst_d3d11_compositor_decide_allocation (GstAggregator * + aggregator, GstQuery * query); + +static GstFlowReturn +gst_d3d11_compositor_aggregate_frames (GstVideoAggregator * vagg, + GstBuffer * outbuf); + +#define gst_d3d11_compositor_parent_class parent_class +G_DEFINE_TYPE_WITH_CODE (GstD3D11Compositor, gst_d3d11_compositor, + GST_TYPE_VIDEO_AGGREGATOR, G_IMPLEMENT_INTERFACE (GST_TYPE_CHILD_PROXY, + gst_d3d11_compositor_child_proxy_init)); + +static void +gst_d3d11_compositor_class_init (GstD3D11CompositorClass * klass) +{ + GObjectClass *gobject_class = G_OBJECT_CLASS (klass); + GstElementClass *element_class = GST_ELEMENT_CLASS (klass); + GstAggregatorClass *aggregator_class = GST_AGGREGATOR_CLASS (klass); + GstVideoAggregatorClass *vagg_class = GST_VIDEO_AGGREGATOR_CLASS (klass); + + gobject_class->dispose = gst_d3d11_compositor_dispose; + gobject_class->set_property = gst_d3d11_compositor_set_property; + gobject_class->get_property = gst_d3d11_compositor_get_property; + + g_object_class_install_property (gobject_class, PROP_ADAPTER, + g_param_spec_int ("adapter", "Adapter", + "Adapter index for creating device (-1 for default)", + -1, G_MAXINT32, DEFAULT_ADAPTER, + G_PARAM_READWRITE | GST_PARAM_MUTABLE_READY | + G_PARAM_STATIC_STRINGS)); + + g_object_class_install_property (gobject_class, PROP_BACKGROUND, + g_param_spec_enum ("background", "Background", "Background type", + GST_TYPE_COMPOSITOR_BACKGROUND, + DEFAULT_BACKGROUND, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + + element_class->request_new_pad = + GST_DEBUG_FUNCPTR (gst_d3d11_compositor_request_new_pad); + element_class->release_pad = + GST_DEBUG_FUNCPTR (gst_d3d11_compositor_release_pad); + element_class->set_context = + GST_DEBUG_FUNCPTR (gst_d3d11_compositor_set_context); + + aggregator_class->start = GST_DEBUG_FUNCPTR (gst_d3d11_compositor_start); + aggregator_class->stop = GST_DEBUG_FUNCPTR (gst_d3d11_compositor_stop); + aggregator_class->sink_query = + GST_DEBUG_FUNCPTR (gst_d3d11_compositor_sink_query); + aggregator_class->src_query = + GST_DEBUG_FUNCPTR (gst_d3d11_compositor_src_query); + aggregator_class->fixate_src_caps = + GST_DEBUG_FUNCPTR (gst_d3d11_compositor_fixate_src_caps); + aggregator_class->propose_allocation = + GST_DEBUG_FUNCPTR (gst_d3d11_compositor_propose_allocation); + aggregator_class->decide_allocation = + GST_DEBUG_FUNCPTR (gst_d3d11_compositor_decide_allocation); + + vagg_class->aggregate_frames = + GST_DEBUG_FUNCPTR (gst_d3d11_compositor_aggregate_frames); + + gst_element_class_add_static_pad_template_with_gtype (element_class, + &src_template, GST_TYPE_AGGREGATOR_PAD); + gst_element_class_add_static_pad_template_with_gtype (element_class, + &sink_template, GST_TYPE_D3D11_COMPOSITOR_PAD); + + gst_element_class_set_static_metadata (element_class, "Direct3D11 Compositor", + "Filter/Editor/Video/Compositor", + "Composite multiple video streams via D3D11 API", + "Seungha Yang "); +} + +static void +gst_d3d11_compositor_init (GstD3D11Compositor * self) +{ + self->adapter = DEFAULT_ADAPTER; + self->background = DEFAULT_BACKGROUND; +} + +static void +gst_d3d11_compositor_dispose (GObject * object) +{ + GstD3D11Compositor *self = GST_D3D11_COMPOSITOR (object); + + gst_clear_object (&self->device); + gst_clear_buffer (&self->fallback_buf); + gst_clear_object (&self->fallback_pool); + g_clear_pointer (&self->checker_background, gst_d3d11_quad_free); + + G_OBJECT_CLASS (parent_class)->dispose (object); +} + +static void +gst_d3d11_compositor_set_property (GObject * object, + guint prop_id, const GValue * value, GParamSpec * pspec) +{ + GstD3D11Compositor *self = GST_D3D11_COMPOSITOR (object); + + switch (prop_id) { + case PROP_ADAPTER: + self->adapter = g_value_get_int (value); + break; + case PROP_BACKGROUND: + self->background = g_value_get_enum (value); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static void +gst_d3d11_compositor_get_property (GObject * object, + guint prop_id, GValue * value, GParamSpec * pspec) +{ + GstD3D11Compositor *self = GST_D3D11_COMPOSITOR (object); + + switch (prop_id) { + case PROP_ADAPTER: + g_value_set_int (value, self->adapter); + break; + case PROP_BACKGROUND: + g_value_set_enum (value, self->background); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static GObject * +gst_d3d11_compositor_child_proxy_get_child_by_index (GstChildProxy * proxy, + guint index) +{ + GstD3D11Compositor *self = GST_D3D11_COMPOSITOR (proxy); + GObject *obj = NULL; + + GST_OBJECT_LOCK (self); + obj = g_list_nth_data (GST_ELEMENT_CAST (self)->sinkpads, index); + if (obj) + gst_object_ref (obj); + GST_OBJECT_UNLOCK (self); + + return obj; +} + +static guint +gst_d3d11_compositor_child_proxy_get_children_count (GstChildProxy * proxy) +{ + GstD3D11Compositor *self = GST_D3D11_COMPOSITOR (proxy); + guint count = 0; + + GST_OBJECT_LOCK (self); + count = GST_ELEMENT_CAST (self)->numsinkpads; + GST_OBJECT_UNLOCK (self); + GST_INFO_OBJECT (self, "Children Count: %d", count); + + return count; +} + +static void +gst_d3d11_compositor_child_proxy_init (gpointer g_iface, gpointer iface_data) +{ + GstChildProxyInterface *iface = g_iface; + + iface->get_child_by_index = + gst_d3d11_compositor_child_proxy_get_child_by_index; + iface->get_children_count = + gst_d3d11_compositor_child_proxy_get_children_count; +} + +static GstPad * +gst_d3d11_compositor_request_new_pad (GstElement * element, + GstPadTemplate * templ, const gchar * name, const GstCaps * caps) +{ + GstPad *pad; + + pad = GST_ELEMENT_CLASS (parent_class)->request_new_pad (element, + templ, name, caps); + + if (pad == NULL) + goto could_not_create; + + gst_child_proxy_child_added (GST_CHILD_PROXY (element), G_OBJECT (pad), + GST_OBJECT_NAME (pad)); + + GST_DEBUG_OBJECT (element, "Created new pad %s:%s", GST_DEBUG_PAD_NAME (pad)); + + return pad; + +could_not_create: + { + GST_DEBUG_OBJECT (element, "could not create/add pad"); + return NULL; + } +} + +static void +gst_d3d11_compositor_release_pad (GstElement * element, GstPad * pad) +{ + GstD3D11Compositor *self = GST_D3D11_COMPOSITOR (element); + GstD3D11CompositorPad *cpad = GST_D3D11_COMPOSITOR_PAD (pad); + + GST_DEBUG_OBJECT (self, "Releasing pad %s:%s", GST_DEBUG_PAD_NAME (pad)); + + gst_child_proxy_child_removed (GST_CHILD_PROXY (self), G_OBJECT (pad), + GST_OBJECT_NAME (pad)); + + gst_clear_buffer (&cpad->fallback_buf); + gst_clear_object (&cpad->fallback_pool); + g_clear_pointer (&cpad->convert, gst_d3d11_color_converter_free); + if (cpad->blend) { + ID3D11BlendState_Release (cpad->blend); + cpad->blend = NULL; + } + + GST_ELEMENT_CLASS (parent_class)->release_pad (element, pad); +} + +static void +gst_d3d11_compositor_set_context (GstElement * element, GstContext * context) +{ + GstD3D11Compositor *self = GST_D3D11_COMPOSITOR (element); + + gst_d3d11_handle_set_context (element, context, self->adapter, &self->device); + + GST_ELEMENT_CLASS (parent_class)->set_context (element, context); +} + +static gboolean +gst_d3d11_compositor_start (GstAggregator * aggregator) +{ + GstD3D11Compositor *self = GST_D3D11_COMPOSITOR (aggregator); + + if (!gst_d3d11_ensure_element_data (GST_ELEMENT_CAST (self), + self->adapter, &self->device)) { + GST_ERROR_OBJECT (self, "Failed to get D3D11 device"); + return FALSE; + } + + return GST_AGGREGATOR_CLASS (parent_class)->start (aggregator); +} + +static gboolean +gst_d3d11_compositor_stop (GstAggregator * aggregator) +{ + GstD3D11Compositor *self = GST_D3D11_COMPOSITOR (aggregator); + + g_clear_pointer (&self->checker_background, gst_d3d11_quad_free); + gst_clear_object (&self->device); + + return GST_AGGREGATOR_CLASS (parent_class)->stop (aggregator); +} + +static GstCaps * +gst_d3d11_compositor_sink_getcaps (GstPad * pad, GstCaps * filter) +{ + GstCaps *sinkcaps; + GstCaps *template_caps; + GstCaps *filtered_caps; + GstCaps *returned_caps; + + template_caps = gst_pad_get_pad_template_caps (pad); + + sinkcaps = gst_pad_get_current_caps (pad); + if (sinkcaps == NULL) { + sinkcaps = gst_caps_ref (template_caps); + } else { + sinkcaps = gst_caps_merge (sinkcaps, gst_caps_ref (template_caps)); + } + + if (filter) { + filtered_caps = gst_caps_intersect (sinkcaps, filter); + gst_caps_unref (sinkcaps); + } else { + filtered_caps = sinkcaps; /* pass ownership */ + } + + returned_caps = gst_caps_intersect (filtered_caps, template_caps); + + gst_caps_unref (template_caps); + gst_caps_unref (filtered_caps); + + GST_DEBUG_OBJECT (pad, "returning %" GST_PTR_FORMAT, returned_caps); + + return returned_caps; +} + +static gboolean +gst_d3d11_compositor_sink_acceptcaps (GstPad * pad, GstCaps * caps) +{ + gboolean ret; + GstCaps *template_caps; + + GST_DEBUG_OBJECT (pad, "try accept caps of %" GST_PTR_FORMAT, caps); + + template_caps = gst_pad_get_pad_template_caps (pad); + template_caps = gst_caps_make_writable (template_caps); + + ret = gst_caps_can_intersect (caps, template_caps); + GST_DEBUG_OBJECT (pad, "%saccepted caps %" GST_PTR_FORMAT, + (ret ? "" : "not "), caps); + gst_caps_unref (template_caps); + + return ret; +} + +static gboolean +gst_d3d11_compositor_sink_query (GstAggregator * aggregator, + GstAggregatorPad * pad, GstQuery * query) +{ + GstD3D11Compositor *self = GST_D3D11_COMPOSITOR (aggregator); + + switch (GST_QUERY_TYPE (query)) { + case GST_QUERY_CONTEXT: + { + gboolean ret; + ret = gst_d3d11_handle_context_query (GST_ELEMENT (aggregator), query, + self->device); + if (ret) + return TRUE; + break; + } + case GST_QUERY_CAPS: + { + GstCaps *filter, *caps; + + gst_query_parse_caps (query, &filter); + caps = gst_d3d11_compositor_sink_getcaps (GST_PAD (pad), filter); + gst_query_set_caps_result (query, caps); + gst_caps_unref (caps); + return TRUE; + } + case GST_QUERY_ACCEPT_CAPS: + { + GstCaps *caps; + gboolean ret; + + gst_query_parse_accept_caps (query, &caps); + ret = gst_d3d11_compositor_sink_acceptcaps (GST_PAD (pad), caps); + gst_query_set_accept_caps_result (query, ret); + return TRUE; + } + default: + break; + } + + return GST_AGGREGATOR_CLASS (parent_class)->sink_query (aggregator, + pad, query); +} + +static gboolean +gst_d3d11_compositor_src_query (GstAggregator * aggregator, GstQuery * query) +{ + GstD3D11Compositor *self = GST_D3D11_COMPOSITOR (aggregator); + + switch (GST_QUERY_TYPE (query)) { + case GST_QUERY_CONTEXT: + { + gboolean ret; + ret = gst_d3d11_handle_context_query (GST_ELEMENT (aggregator), query, + self->device); + if (ret) + return TRUE; + break; + } + default: + break; + } + + return GST_AGGREGATOR_CLASS (parent_class)->src_query (aggregator, query); +} + +static GstCaps * +gst_d3d11_compositor_fixate_src_caps (GstAggregator * aggregator, + GstCaps * caps) +{ + GstVideoAggregator *vagg = GST_VIDEO_AGGREGATOR (aggregator); + GList *l; + gint best_width = -1, best_height = -1; + gint best_fps_n = -1, best_fps_d = -1; + gint par_n, par_d; + gdouble best_fps = 0.; + GstCaps *ret = NULL; + GstStructure *s; + + ret = gst_caps_make_writable (caps); + + /* we need this to calculate how large to make the output frame */ + s = gst_caps_get_structure (ret, 0); + if (gst_structure_has_field (s, "pixel-aspect-ratio")) { + gst_structure_fixate_field_nearest_fraction (s, "pixel-aspect-ratio", 1, 1); + gst_structure_get_fraction (s, "pixel-aspect-ratio", &par_n, &par_d); + } else { + par_n = par_d = 1; + } + + GST_OBJECT_LOCK (vagg); + for (l = GST_ELEMENT (vagg)->sinkpads; l; l = l->next) { + GstVideoAggregatorPad *vaggpad = l->data; + GstD3D11CompositorPad *cpad = GST_D3D11_COMPOSITOR_PAD (vaggpad); + gint this_width, this_height; + gint width, height; + gint fps_n, fps_d; + gdouble cur_fps; + + fps_n = GST_VIDEO_INFO_FPS_N (&vaggpad->info); + fps_d = GST_VIDEO_INFO_FPS_D (&vaggpad->info); + gst_d3d11_compositor_pad_get_output_size (cpad, + par_n, par_d, &width, &height); + + if (width == 0 || height == 0) + continue; + + this_width = width + MAX (cpad->xpos, 0); + this_height = height + MAX (cpad->ypos, 0); + + if (best_width < this_width) + best_width = this_width; + if (best_height < this_height) + best_height = this_height; + + if (fps_d == 0) + cur_fps = 0.0; + else + gst_util_fraction_to_double (fps_n, fps_d, &cur_fps); + + if (best_fps < cur_fps) { + best_fps = cur_fps; + best_fps_n = fps_n; + best_fps_d = fps_d; + } + } + GST_OBJECT_UNLOCK (vagg); + + if (best_fps_n <= 0 || best_fps_d <= 0 || best_fps == 0.0) { + best_fps_n = 25; + best_fps_d = 1; + best_fps = 25.0; + } + + gst_structure_fixate_field_nearest_int (s, "width", best_width); + gst_structure_fixate_field_nearest_int (s, "height", best_height); + gst_structure_fixate_field_nearest_fraction (s, "framerate", best_fps_n, + best_fps_d); + ret = gst_caps_fixate (ret); + + GST_LOG_OBJECT (aggregator, "Fixated caps %" GST_PTR_FORMAT, ret); + + return ret; +} + +static gboolean +gst_d3d11_compositor_propose_allocation (GstAggregator * aggregator, + GstAggregatorPad * pad, GstQuery * decide_query, GstQuery * query) +{ + GstD3D11Compositor *self = GST_D3D11_COMPOSITOR (aggregator); + GstVideoInfo info; + GstBufferPool *pool; + GstCaps *caps; + guint size; + + gst_query_parse_allocation (query, &caps, NULL); + + if (caps == NULL) + return FALSE; + + if (!gst_video_info_from_caps (&info, caps)) + return FALSE; + + if (gst_query_get_n_allocation_pools (query) == 0) { + GstStructure *config; + GstD3D11AllocationParams *d3d11_params; + gint i; + + pool = gst_d3d11_buffer_pool_new (self->device); + config = gst_buffer_pool_get_config (pool); + + gst_buffer_pool_config_add_option (config, + GST_BUFFER_POOL_OPTION_VIDEO_META); + + d3d11_params = gst_buffer_pool_config_get_d3d11_allocation_params (config); + if (!d3d11_params) { + d3d11_params = gst_d3d11_allocation_params_new (self->device, &info, 0, + D3D11_BIND_SHADER_RESOURCE); + } else { + /* Set bind flag */ + for (i = 0; i < GST_VIDEO_INFO_N_PLANES (&info); i++) { + d3d11_params->desc[i].BindFlags |= D3D11_BIND_SHADER_RESOURCE; + } + } + + gst_buffer_pool_config_set_d3d11_allocation_params (config, d3d11_params); + gst_d3d11_allocation_params_free (d3d11_params); + + size = GST_VIDEO_INFO_SIZE (&info); + gst_buffer_pool_config_set_params (config, caps, size, 0, 0); + + if (!gst_buffer_pool_set_config (pool, config)) + goto config_failed; + + /* d3d11 buffer pool might update buffer size by self */ + size = GST_D3D11_BUFFER_POOL (pool)->buffer_size; + + gst_query_add_allocation_pool (query, pool, size, 0, 0); + gst_object_unref (pool); + } + + gst_query_add_allocation_meta (query, GST_VIDEO_META_API_TYPE, NULL); + + return TRUE; + + /* ERRORS */ +config_failed: + { + GST_ERROR_OBJECT (self, "failed to set config"); + gst_object_unref (pool); + return FALSE; + } +} + +static gboolean +gst_d3d11_compositor_decide_allocation (GstAggregator * aggregator, + GstQuery * query) +{ + GstD3D11Compositor *self = GST_D3D11_COMPOSITOR (aggregator); + GstCaps *caps; + GstBufferPool *pool = NULL; + guint n, size, min, max; + GstVideoInfo info; + GstStructure *config; + GstD3D11AllocationParams *d3d11_params; + + gst_query_parse_allocation (query, &caps, NULL); + + if (!caps) { + GST_DEBUG_OBJECT (self, "No output caps"); + return FALSE; + } + + gst_video_info_from_caps (&info, caps); + n = gst_query_get_n_allocation_pools (query); + if (n > 0) + gst_query_parse_nth_allocation_pool (query, 0, &pool, &size, &min, &max); + + /* create our own pool */ + if (pool && !GST_D3D11_BUFFER_POOL (pool)) { + gst_object_unref (pool); + pool = NULL; + } + + if (!pool) { + pool = gst_d3d11_buffer_pool_new (self->device); + + min = max = 0; + size = (guint) info.size; + } + + config = gst_buffer_pool_get_config (pool); + gst_buffer_pool_config_set_params (config, caps, size, min, max); + gst_buffer_pool_config_add_option (config, GST_BUFFER_POOL_OPTION_VIDEO_META); + + d3d11_params = gst_buffer_pool_config_get_d3d11_allocation_params (config); + if (!d3d11_params) { + d3d11_params = gst_d3d11_allocation_params_new (self->device, + &info, 0, D3D11_BIND_RENDER_TARGET); + } else { + gint i; + + for (i = 0; i < GST_VIDEO_INFO_N_PLANES (&info); i++) { + d3d11_params->desc[i].BindFlags |= D3D11_BIND_RENDER_TARGET; + } + } + + gst_buffer_pool_config_set_d3d11_allocation_params (config, d3d11_params); + gst_d3d11_allocation_params_free (d3d11_params); + + gst_buffer_pool_set_config (pool, config); + /* d3d11 buffer pool might update buffer size by self */ + size = GST_D3D11_BUFFER_POOL (pool)->buffer_size; + + if (n > 0) + gst_query_set_nth_allocation_pool (query, 0, pool, size, min, max); + else + gst_query_add_allocation_pool (query, pool, size, min, max); + gst_object_unref (pool); + + return TRUE; +} + +typedef struct +{ + struct + { + FLOAT x; + FLOAT y; + FLOAT z; + } position; + struct + { + FLOAT u; + FLOAT v; + } texture; +} VertexData; + +static GstD3D11Quad * +gst_d3d11_compositor_create_checker_quad (GstD3D11Compositor * self) +{ + ID3D11PixelShader *ps = NULL; + ID3D11VertexShader *vs = NULL; + ID3D11InputLayout *layout = NULL; + GstD3D11Quad *quad = NULL; + VertexData *vertex_data; + WORD *indices; + ID3D11Device *device_handle; + ID3D11DeviceContext *context_handle; + D3D11_MAPPED_SUBRESOURCE map; + D3D11_INPUT_ELEMENT_DESC input_desc; + D3D11_BUFFER_DESC buffer_desc; + ID3D11Buffer *vertex_buffer = NULL; + ID3D11Buffer *index_buffer = NULL; + gboolean ret = FALSE; + HRESULT hr; + + device_handle = gst_d3d11_device_get_device_handle (self->device); + context_handle = gst_d3d11_device_get_device_context_handle (self->device); + + ret = gst_d3d11_create_pixel_shader (self->device, checker_ps_src, &ps); + if (!ret) { + GST_ERROR_OBJECT (self, "Couldn't setup pixel shader"); + goto done; + } + + memset (&input_desc, 0, sizeof (D3D11_INPUT_ELEMENT_DESC)); + input_desc.SemanticName = "POSITION"; + input_desc.SemanticIndex = 0; + input_desc.Format = DXGI_FORMAT_R32G32B32_FLOAT; + input_desc.InputSlot = 0; + input_desc.AlignedByteOffset = D3D11_APPEND_ALIGNED_ELEMENT; + input_desc.InputSlotClass = D3D11_INPUT_PER_VERTEX_DATA; + input_desc.InstanceDataStepRate = 0; + + if (!gst_d3d11_create_vertex_shader (self->device, checker_vs_src, + &input_desc, 1, &vs, &layout)) { + GST_ERROR_OBJECT (self, "Couldn't setup vertex shader"); + goto done; + } + + memset (&buffer_desc, 0, sizeof (D3D11_BUFFER_DESC)); + buffer_desc.Usage = D3D11_USAGE_DYNAMIC; + buffer_desc.ByteWidth = sizeof (VertexData) * 4; + buffer_desc.BindFlags = D3D11_BIND_VERTEX_BUFFER; + buffer_desc.CPUAccessFlags = D3D11_CPU_ACCESS_WRITE; + + hr = ID3D11Device_CreateBuffer (device_handle, &buffer_desc, NULL, + &vertex_buffer); + + if (!gst_d3d11_result (hr, self->device)) { + GST_ERROR_OBJECT (self, + "Couldn't create vertex buffer, hr: 0x%x", (guint) hr); + goto done; + } + + hr = ID3D11DeviceContext_Map (context_handle, + (ID3D11Resource *) vertex_buffer, 0, D3D11_MAP_WRITE_DISCARD, 0, &map); + + if (!gst_d3d11_result (hr, self->device)) { + GST_ERROR_OBJECT (self, "Couldn't map vertex buffer, hr: 0x%x", (guint) hr); + goto done; + } + + vertex_data = (VertexData *) map.pData; + /* bottom left */ + /* bottom left */ + vertex_data[0].position.x = -1.0f; + vertex_data[0].position.y = -1.0f; + vertex_data[0].position.z = 0.0f; + vertex_data[0].texture.u = 0.0f; + vertex_data[0].texture.v = 1.0f; + + /* top left */ + vertex_data[1].position.x = -1.0f; + vertex_data[1].position.y = 1.0f; + vertex_data[1].position.z = 0.0f; + vertex_data[1].texture.u = 0.0f; + vertex_data[1].texture.v = 0.0f; + + /* top right */ + vertex_data[2].position.x = 1.0f; + vertex_data[2].position.y = 1.0f; + vertex_data[2].position.z = 0.0f; + vertex_data[2].texture.u = 1.0f; + vertex_data[2].texture.v = 0.0f; + + /* bottom right */ + vertex_data[3].position.x = 1.0f; + vertex_data[3].position.y = -1.0f; + vertex_data[3].position.z = 0.0f; + vertex_data[3].texture.u = 1.0f; + vertex_data[3].texture.v = 1.0f; + + ID3D11DeviceContext_Unmap (context_handle, + (ID3D11Resource *) vertex_buffer, 0); + + buffer_desc.Usage = D3D11_USAGE_DYNAMIC; + buffer_desc.ByteWidth = sizeof (WORD) * 6; + buffer_desc.BindFlags = D3D11_BIND_INDEX_BUFFER; + buffer_desc.CPUAccessFlags = D3D11_CPU_ACCESS_WRITE; + + hr = ID3D11Device_CreateBuffer (device_handle, &buffer_desc, NULL, + &index_buffer); + + if (!gst_d3d11_result (hr, self->device)) { + GST_ERROR_OBJECT (self, + "Couldn't create index buffer, hr: 0x%x", (guint) hr); + goto done; + } + + hr = ID3D11DeviceContext_Map (context_handle, + (ID3D11Resource *) index_buffer, 0, D3D11_MAP_WRITE_DISCARD, 0, &map); + + if (!gst_d3d11_result (hr, self->device)) { + GST_ERROR_OBJECT (self, "Couldn't map index buffer, hr: 0x%x", (guint) hr); + goto done; + } + + indices = (WORD *) map.pData; + + /* clockwise indexing */ + indices[0] = 0; /* bottom left */ + indices[1] = 1; /* top left */ + indices[2] = 2; /* top right */ + + indices[3] = 3; /* bottom right */ + indices[4] = 0; /* bottom left */ + indices[5] = 2; /* top right */ + + ID3D11DeviceContext_Unmap (context_handle, + (ID3D11Resource *) index_buffer, 0); + + quad = gst_d3d11_quad_new (self->device, + ps, vs, layout, NULL, NULL, NULL, NULL, vertex_buffer, + sizeof (VertexData), index_buffer, DXGI_FORMAT_R16_UINT, 6); + if (!quad) { + GST_ERROR_OBJECT (self, "Couldn't setup quad"); + goto done; + } + +done: + if (ps) + ID3D11PixelShader_Release (ps); + if (vs) + ID3D11VertexShader_Release (vs); + if (layout) + ID3D11InputLayout_Release (layout); + if (vertex_buffer) + ID3D11Buffer_Release (vertex_buffer); + if (index_buffer) + ID3D11Buffer_Release (index_buffer); + + return quad; +} + +static gboolean +gst_d3d11_compositor_draw_background_checker (GstD3D11Compositor * self, + ID3D11RenderTargetView * rtv) +{ + if (!self->checker_background) { + GstVideoInfo *info = &GST_VIDEO_AGGREGATOR_CAST (self)->info; + + self->checker_background = gst_d3d11_compositor_create_checker_quad (self); + + if (!self->checker_background) + return FALSE; + + self->viewport.TopLeftX = 0; + self->viewport.TopLeftY = 0; + self->viewport.Width = GST_VIDEO_INFO_WIDTH (info); + self->viewport.Height = GST_VIDEO_INFO_HEIGHT (info); + self->viewport.MinDepth = 0.0f; + self->viewport.MaxDepth = 1.0f; + } + + return gst_d3d11_draw_quad_unlocked (self->checker_background, + &self->viewport, 1, NULL, 0, &rtv, 1, NULL, NULL, NULL); +} + +/* Must be called with d3d11 device lock */ +static gboolean +gst_d3d11_compositor_draw_background (GstD3D11Compositor * self, + ID3D11RenderTargetView * rtv) +{ + ID3D11DeviceContext *device_context = + gst_d3d11_device_get_device_context_handle (self->device); + FLOAT rgba[4] = { 0.0f, 0.0f, 0.0f, 1.0f }; + + switch (self->background) { + case GST_D3D11_COMPOSITOR_BACKGROUND_CHECKER: + return gst_d3d11_compositor_draw_background_checker (self, rtv); + case GST_D3D11_COMPOSITOR_BACKGROUND_BLACK: + /* {0, 0, 0, 1} */ + break; + case GST_D3D11_COMPOSITOR_BACKGROUND_WHITE: + rgba[0] = 1.0f; + rgba[1] = 1.0f; + rgba[2] = 1.0f; + break; + case GST_D3D11_COMPOSITOR_BACKGROUND_TRANSPARENT: + rgba[3] = 0.0f; + break; + default: + g_assert_not_reached (); + return FALSE; + } + + ID3D11DeviceContext_ClearRenderTargetView (device_context, rtv, rgba); + + return TRUE; +} + +static GstFlowReturn +gst_d3d11_compositor_aggregate_frames (GstVideoAggregator * vagg, + GstBuffer * outbuf) +{ + GstD3D11Compositor *self = GST_D3D11_COMPOSITOR (vagg); + GList *iter; + GstBuffer *target_buf = outbuf; + gboolean need_copy = FALSE; + gboolean do_device_copy = FALSE; + GstFlowReturn ret = GST_FLOW_OK; + ID3D11RenderTargetView *rtv[GST_VIDEO_MAX_PLANES] = { NULL, }; + gint i, j; + gint view_idx; + + /* Use fallback buffer when output buffer is: + * - non-d3d11 memory + * - or, from different d3d11 device + * - or not bound to render target + */ + if (!gst_d3d11_compositor_check_d3d11_memory (self, + outbuf, FALSE, &do_device_copy) || !do_device_copy) { + if (!gst_d3d11_compsitor_prepare_fallback_buffer (self, &vagg->info, FALSE, + &self->fallback_pool, &self->fallback_buf)) { + GST_ERROR_OBJECT (self, "Couldn't prepare fallback buffer"); + return GST_FLOW_ERROR; + } + + GST_TRACE_OBJECT (self, "Will draw on fallback texture"); + + need_copy = TRUE; + target_buf = self->fallback_buf; + } + + view_idx = 0; + for (i = 0; i < gst_buffer_n_memory (target_buf); i++) { + GstMemory *mem = gst_buffer_peek_memory (target_buf, i); + GstD3D11Memory *dmem; + + if (!gst_is_d3d11_memory (mem)) { + GST_ERROR_OBJECT (self, "Invalid output memory"); + return GST_FLOW_ERROR; + } + + dmem = (GstD3D11Memory *) mem; + if (!gst_d3d11_memory_ensure_render_target_view (dmem)) { + GST_ERROR_OBJECT (self, "Render target view is unavailable"); + return GST_FLOW_ERROR; + } + + for (j = 0; j < dmem->num_render_target_views; j++) { + g_assert (view_idx < GST_VIDEO_MAX_PLANES); + + rtv[view_idx] = dmem->render_target_view[j]; + view_idx++; + } + + /* Mark need-download for fallback buffer use case */ + GST_MINI_OBJECT_FLAG_SET (dmem, GST_D3D11_MEMORY_TRANSFER_NEED_DOWNLOAD); + } + + gst_d3d11_device_lock (self->device); + /* XXX: the number of render target view must be one here, since we support + * only RGBA or BGRA */ + if (!gst_d3d11_compositor_draw_background (self, rtv[0])) { + GST_ERROR_OBJECT (self, "Couldn't draw background"); + gst_d3d11_device_unlock (self->device); + ret = GST_FLOW_ERROR; + goto done; + } + + GST_OBJECT_LOCK (self); + + for (iter = GST_ELEMENT (vagg)->sinkpads; iter; iter = g_list_next (iter)) { + GstVideoAggregatorPad *pad = iter->data; + GstD3D11CompositorPad *cpad = GST_D3D11_COMPOSITOR_PAD (pad); + GstVideoFrame *prepared_frame = + gst_video_aggregator_pad_get_prepared_frame (pad); + ID3D11ShaderResourceView *srv[GST_VIDEO_MAX_PLANES] = { NULL, }; + GstBuffer *buffer; + + if (!prepared_frame) + continue; + + if (!gst_d3d11_compositor_pad_setup_converter (pad, vagg)) { + GST_ERROR_OBJECT (self, "Couldn't setup converter"); + ret = GST_FLOW_ERROR; + break; + } + + buffer = prepared_frame->buffer; + + view_idx = 0; + for (i = 0; i < gst_buffer_n_memory (buffer); i++) { + GstD3D11Memory *dmem = + (GstD3D11Memory *) gst_buffer_peek_memory (buffer, i); + + for (j = 0; j < dmem->num_shader_resource_views; j++) { + g_assert (view_idx < GST_VIDEO_MAX_PLANES); + + srv[view_idx] = dmem->shader_resource_view[j]; + view_idx++; + } + } + + if (!gst_d3d11_color_converter_convert_unlocked (cpad->convert, srv, rtv, + cpad->blend, cpad->blend_factor)) { + GST_ERROR_OBJECT (self, "Couldn't convert frame"); + ret = GST_FLOW_ERROR; + break; + } + } + GST_OBJECT_UNLOCK (self); + gst_d3d11_device_unlock (self->device); + + if (ret != GST_FLOW_OK) + goto done; + + if (need_copy && !gst_d3d11_compositor_copy_buffer (self, &vagg->info, + target_buf, outbuf, do_device_copy)) { + GST_ERROR_OBJECT (self, "Couldn't copy input buffer to fallback buffer"); + ret = GST_FLOW_ERROR; + } + +done: + gst_clear_buffer (&self->fallback_buf); + + return ret; +} diff --git a/sys/d3d11/gstd3d11compositor.h b/sys/d3d11/gstd3d11compositor.h new file mode 100644 index 0000000000..d0b6750bbe --- /dev/null +++ b/sys/d3d11/gstd3d11compositor.h @@ -0,0 +1,83 @@ +/* + * GStreamer + * Copyright (C) 2020 Seungha Yang + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifndef __GST_D3D11_COMPOSITOR_H__ +#define __GST_D3D11_COMPOSITOR_H__ + +#include +#include +#include + +G_BEGIN_DECLS + +#define GST_TYPE_D3D11_COMPOSITOR_PAD (gst_d3d11_compositor_pad_get_type()) +G_DECLARE_FINAL_TYPE (GstD3D11CompositorPad, gst_d3d11_compositor_pad, + GST, D3D11_COMPOSITOR_PAD, GstVideoAggregatorPad) + +#define GST_TYPE_D3D11_COMPOSITOR (gst_d3d11_compositor_get_type()) +G_DECLARE_FINAL_TYPE (GstD3D11Compositor, gst_d3d11_compositor, + GST, D3D11_COMPOSITOR, GstVideoAggregator) + +typedef enum +{ + GST_D3D11_COMPOSITOR_BLEND_OP_ADD, + GST_D3D11_COMPOSITOR_BLEND_OP_SUBTRACT, + GST_D3D11_COMPOSITOR_BLEND_OP_REV_SUBTRACT, + GST_D3D11_COMPOSITOR_BLEND_OP_MIN, + GST_D3D11_COMPOSITOR_BLEND_OP_MAX +} GstD3D11CompositorBlendOperation; + +#define GST_TYPE_D3D11_COMPOSITOR_BLEND_OPERATION (gst_d3d11_compositor_blend_operation_get_type()) +GType gst_d3d11_compositor_blend_operation_get_type (void); + +typedef enum +{ + GST_D3D11_COMPOSITOR_BLEND_ZERO, + GST_D3D11_COMPOSITOR_BLEND_ONE, + GST_D3D11_COMPOSITOR_BLEND_SRC_COLOR, + GST_D3D11_COMPOSITOR_BLEND_INV_SRC_COLOR, + GST_D3D11_COMPOSITOR_BLEND_SRC_ALPHA, + GST_D3D11_COMPOSITOR_BLEND_INV_SRC_ALPHA, + GST_D3D11_COMPOSITOR_BLEND_DEST_ALPHA, + GST_D3D11_COMPOSITOR_BLEND_INV_DEST_ALPHA, + GST_D3D11_COMPOSITOR_BLEND_DEST_COLOR, + GST_D3D11_COMPOSITOR_BLEND_INV_DEST_COLOR, + GST_D3D11_COMPOSITOR_BLEND_SRC_ALPHA_SAT, + GST_D3D11_COMPOSITOR_BLEND_BLEND_FACTOR, + GST_D3D11_COMPOSITOR_BLEND_INV_BLEND_FACTOR, +} GstD3D11CompositorBlend; + +#define GST_TYPE_D3D11_COMPOSITOR_BLEND (gst_d3d11_compositor_blend_get_type()) +GType gst_d3d11_compositor_blend_get_type (void); + +typedef enum +{ + GST_D3D11_COMPOSITOR_BACKGROUND_CHECKER, + GST_D3D11_COMPOSITOR_BACKGROUND_BLACK, + GST_D3D11_COMPOSITOR_BACKGROUND_WHITE, + GST_D3D11_COMPOSITOR_BACKGROUND_TRANSPARENT, +} GstD3D11CompositorBackground; + +#define GST_TYPE_COMPOSITOR_BACKGROUND (gst_d3d11_compositor_background_get_type()) +GType gst_d3d11_compositor_background_get_type (void); + +G_END_DECLS + +#endif /* __GST_D3D11_COMPOSITOR_H__ */ diff --git a/sys/d3d11/gstd3d11compositorbin.c b/sys/d3d11/gstd3d11compositorbin.c new file mode 100644 index 0000000000..792b8ea75f --- /dev/null +++ b/sys/d3d11/gstd3d11compositorbin.c @@ -0,0 +1,948 @@ +/* + * GStreamer + * Copyright (C) 2015 Matthew Waters + * Copyright (C) 2020 Seungha Yang + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include "gstd3d11_fwd.h" +#include "gstd3d11compositorbin.h" +#include "gstd3d11compositor.h" +#include "gstd3d11device.h" +#include "gstd3d11memory.h" +#include "gstd3d11bufferpool.h" +#include "gstd3d11utils.h" +#include "gstd3d11format.h" + +GST_DEBUG_CATEGORY_EXTERN (gst_d3d11_compositor_debug); +#define GST_CAT_DEFAULT gst_d3d11_compositor_debug + +/**************************** + * GstD3D11CompositorBinPad * + ****************************/ + +enum +{ + PROP_PAD_0, + /* GstAggregatorPad */ + PROP_PAD_EMIT_SIGNALS, +}; + +/* GstAggregatorPad */ +#define DEFAULT_PAD_EMIT_SIGNALS FALSE + +enum +{ + /* GstAggregatorPad */ + SIGNAL_PAD_BUFFER_CONSUMED = 0, + SIGNAL_PAD_LAST, +}; + +static guint gst_d3d11_compositor_bin_pad_signals[SIGNAL_PAD_LAST] = { 0 }; + +struct _GstD3D11CompositorBinPad +{ + GstGhostPad parent; + + /* Holds ref */ + GstPad *target; + gulong sig_id; +}; + +static void gst_d3d11_compositor_bin_pad_dispose (GObject * object); +static void gst_d3d11_compositor_bin_pad_set_property (GObject * object, + guint prop_id, const GValue * value, GParamSpec * pspec); +static void gst_d3d11_compositor_bin_pad_get_property (GObject * object, + guint prop_id, GValue * value, GParamSpec * pspec); +static void +gst_d3d11_compositor_bin_pad_set_target_default (GstD3D11CompositorBinPad * pad, + GstPad * target); + +G_DEFINE_TYPE (GstD3D11CompositorBinPad, gst_d3d11_compositor_bin_pad, + GST_TYPE_GHOST_PAD); + +static void +gst_d3d11_compositor_bin_pad_class_init (GstD3D11CompositorBinPadClass * klass) +{ + GObjectClass *gobject_class = G_OBJECT_CLASS (klass); + + gobject_class->dispose = gst_d3d11_compositor_bin_pad_dispose; + gobject_class->set_property = gst_d3d11_compositor_bin_pad_set_property; + gobject_class->get_property = gst_d3d11_compositor_bin_pad_get_property; + + /* GstAggregatorPad */ + g_object_class_install_property (gobject_class, PROP_PAD_EMIT_SIGNALS, + g_param_spec_boolean ("emit-signals", "Emit signals", + "Send signals to signal data consumption", + DEFAULT_PAD_EMIT_SIGNALS, + G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + + gst_d3d11_compositor_bin_pad_signals[SIGNAL_PAD_BUFFER_CONSUMED] = + g_signal_new ("buffer-consumed", G_TYPE_FROM_CLASS (klass), + G_SIGNAL_RUN_FIRST, 0, NULL, NULL, NULL, G_TYPE_NONE, 1, GST_TYPE_BUFFER); + + klass->set_target = + GST_DEBUG_FUNCPTR (gst_d3d11_compositor_bin_pad_set_target_default); +} + +static void +gst_d3d11_compositor_bin_pad_init (GstD3D11CompositorBinPad * self) +{ +} + +static void +gst_d3d11_compositor_bin_pad_dispose (GObject * object) +{ + GstD3D11CompositorBinPad *self = GST_D3D11_COMPOSITOR_BIN_PAD (object); + + gst_clear_object (&self->target); + + G_OBJECT_CLASS (gst_d3d11_compositor_bin_pad_parent_class)->dispose (object); +} + +static void +gst_d3d11_compositor_bin_pad_set_property (GObject * object, + guint prop_id, const GValue * value, GParamSpec * pspec) +{ + GstD3D11CompositorBinPad *self = GST_D3D11_COMPOSITOR_BIN_PAD (object); + + if (self->target) + g_object_set_property (G_OBJECT (self->target), pspec->name, value); +} + +static void +gst_d3d11_compositor_bin_pad_get_property (GObject * object, + guint prop_id, GValue * value, GParamSpec * pspec) +{ + GstD3D11CompositorBinPad *self = GST_D3D11_COMPOSITOR_BIN_PAD (object); + + if (self->target) + g_object_get_property (G_OBJECT (self->target), pspec->name, value); +} + +static void +gst_d3d11_compositor_bin_pad_on_buffer_consumed (GstAggregatorPad * pad, + GstBuffer * buffer, GstD3D11CompositorBinPad * self) +{ + g_signal_emit (self, + gst_d3d11_compositor_bin_pad_signals[SIGNAL_PAD_BUFFER_CONSUMED], + 0, buffer); +} + +/** + * gst_d3d11_compositor_bin_pad_set_target: + * @self: a #GstD3D11CompositorBinPad + * @target: (transfer full): a #GstAggregatorPad + */ +static void +gst_d3d11_compositor_bin_pad_set_target (GstD3D11CompositorBinPad * pad, + GstPad * target) +{ + GstD3D11CompositorBinPadClass *klass = + GST_D3D11_COMPOSITOR_BIN_PAD_GET_CLASS (pad); + + klass->set_target (pad, target); +} + +static void +gst_d3d11_compositor_bin_pad_set_target_default (GstD3D11CompositorBinPad * pad, + GstPad * target) +{ + pad->target = target; + pad->sig_id = g_signal_connect (target, "buffer-consumed", + G_CALLBACK (gst_d3d11_compositor_bin_pad_on_buffer_consumed), pad); +} + +static void +gst_d3d11_compositor_bin_pad_unset_target (GstD3D11CompositorBinPad * self) +{ + if (!self->target) + return; + + if (self->sig_id) + g_signal_handler_disconnect (self->target, self->sig_id); + self->sig_id = 0; + gst_clear_object (&self->target); +} + +/****************************** + * GstD3D11CompositorBinInput * + ******************************/ + +enum +{ + PROP_INPUT_0, + /* GstVideoAggregatorPad */ + PROP_INPUT_ZORDER, + PROP_INPUT_REPEAT_AFTER_EOS, + PROP_INPUT_MAX_LAST_BUFFER_REPEAT, + /* GstD3D11CompositorPad */ + PROP_INPUT_XPOS, + PROP_INPUT_YPOS, + PROP_INPUT_WIDTH, + PROP_INPUT_HEIGHT, + PROP_INPUT_ALPHA, + PROP_INPUT_BLEND_OP_RGB, + PROP_INPUT_BLEND_OP_ALPHA, + PROP_INPUT_BLEND_SRC_RGB, + PROP_INPUT_BLEND_SRC_ALPHA, + PROP_INPUT_BLEND_DEST_RGB, + PROP_INPUT_BLEND_DEST_ALPHA, + PROP_INPUT_BLEND_FACTOR_RED, + PROP_INPUT_BLEND_FACTOR_GREEN, + PROP_INPUT_BLEND_FACTOR_BLUE, + PROP_INPUT_BLEND_FACTOR_ALPHA, +}; + +/* GstVideoAggregatorPad */ +#define DEFAULT_INPUT_ZORDER 0 +#define DEFAULT_INPUT_REPEAT_AFTER_EOS FALSE +#define DEFAULT_INPUT_MAX_LAST_BUFFER_REPEAT GST_CLOCK_TIME_NONE +/* GstD3D11CompositorPad */ +#define DEFAULT_INPUT_XPOS 0 +#define DEFAULT_INPUT_YPOS 0 +#define DEFAULT_INPUT_WIDTH 0 +#define DEFAULT_INPUT_HEIGHT 0 +#define DEFAULT_INPUT_ALPHA 1.0 +#define DEFAULT_INPUT_BLEND_OP_RGB GST_D3D11_COMPOSITOR_BLEND_OP_ADD +#define DEFAULT_INPUT_BLEND_OP_ALPHA GST_D3D11_COMPOSITOR_BLEND_OP_ADD +#define DEFAULT_INPUT_BLEND_SRC_RGB GST_D3D11_COMPOSITOR_BLEND_SRC_ALPHA +#define DEFAULT_INPUT_BLEND_SRC_ALPHA GST_D3D11_COMPOSITOR_BLEND_ONE +#define DEFAULT_INPUT_BLEND_DEST_RGB GST_D3D11_COMPOSITOR_BLEND_INV_SRC_ALPHA +#define DEFAULT_INPUT_BLEND_DEST_ALPHA GST_D3D11_COMPOSITOR_BLEND_INV_SRC_ALPHA + +struct _GstD3D11CompositorBinInput +{ + GstD3D11CompositorBinPad parent; +}; + +static void gst_d3d11_compositor_bin_input_set_property (GObject * object, + guint prop_id, const GValue * value, GParamSpec * pspec); +static void gst_d3d11_compositor_bin_input_get_property (GObject * object, + guint prop_id, GValue * value, GParamSpec * pspec); +static void +gst_d3d11_compositor_bin_input_set_target (GstD3D11CompositorBinPad * pad, + GstPad * target); + +#define gst_d3d11_compositor_bin_input_parent_class input_parent_class +G_DEFINE_TYPE (GstD3D11CompositorBinInput, gst_d3d11_compositor_bin_input, + GST_TYPE_D3D11_COMPOSITOR_BIN_PAD); + +static void +gst_d3d11_compositor_bin_input_class_init (GstD3D11CompositorBinInputClass * + klass) +{ + GObjectClass *gobject_class = G_OBJECT_CLASS (klass); + GstD3D11CompositorBinPadClass *pad_class = + GST_D3D11_COMPOSITOR_BIN_PAD_CLASS (klass); + + gobject_class->set_property = gst_d3d11_compositor_bin_input_set_property; + gobject_class->get_property = gst_d3d11_compositor_bin_input_get_property; + + /* GstVideoAggregatorPad */ + g_object_class_install_property (gobject_class, PROP_INPUT_ZORDER, + g_param_spec_uint ("zorder", "Z-Order", "Z Order of the picture", + 0, G_MAXUINT, DEFAULT_INPUT_ZORDER, + G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); + + g_object_class_install_property (gobject_class, PROP_INPUT_REPEAT_AFTER_EOS, + g_param_spec_boolean ("repeat-after-eos", "Repeat After EOS", + "Repeat the " "last frame after EOS until all pads are EOS", + DEFAULT_INPUT_REPEAT_AFTER_EOS, + G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); + + g_object_class_install_property (gobject_class, + PROP_INPUT_MAX_LAST_BUFFER_REPEAT, + g_param_spec_uint64 ("max-last-buffer-repeat", "Max Last Buffer Repeat", + "Repeat last buffer for time (in ns, -1=until EOS), " + "behaviour on EOS is not affected", 0, G_MAXUINT64, + DEFAULT_INPUT_MAX_LAST_BUFFER_REPEAT, + G_PARAM_READWRITE | GST_PARAM_MUTABLE_PLAYING | + G_PARAM_STATIC_STRINGS)); + + /* GstD3D11CompositorPad */ + g_object_class_install_property (gobject_class, PROP_INPUT_XPOS, + g_param_spec_int ("xpos", "X Position", "X position of the picture", + G_MININT, G_MAXINT, DEFAULT_INPUT_XPOS, + G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); + + g_object_class_install_property (gobject_class, PROP_INPUT_YPOS, + g_param_spec_int ("ypos", "Y Position", "Y position of the picture", + G_MININT, G_MAXINT, DEFAULT_INPUT_YPOS, + G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); + + g_object_class_install_property (gobject_class, PROP_INPUT_WIDTH, + g_param_spec_int ("width", "Width", "Width of the picture", + G_MININT, G_MAXINT, DEFAULT_INPUT_WIDTH, + G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); + + g_object_class_install_property (gobject_class, PROP_INPUT_HEIGHT, + g_param_spec_int ("height", "Height", "Height of the picture", + G_MININT, G_MAXINT, DEFAULT_INPUT_HEIGHT, + G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); + + g_object_class_install_property (gobject_class, PROP_INPUT_ALPHA, + g_param_spec_double ("alpha", "Alpha", "Alpha of the picture", 0.0, 1.0, + DEFAULT_INPUT_ALPHA, + G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); + + g_object_class_install_property (gobject_class, PROP_INPUT_BLEND_OP_RGB, + g_param_spec_enum ("blend-op-rgb", "Blend Operation RGB", + "Blend equation for RGB", GST_TYPE_D3D11_COMPOSITOR_BLEND_OPERATION, + DEFAULT_INPUT_BLEND_OP_RGB, + G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); + + g_object_class_install_property (gobject_class, PROP_INPUT_BLEND_OP_ALPHA, + g_param_spec_enum ("blend-op-alpha", "Blend Operation Alpha", + "Blend equation for alpha", GST_TYPE_D3D11_COMPOSITOR_BLEND_OPERATION, + DEFAULT_INPUT_BLEND_OP_ALPHA, + G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); + + g_object_class_install_property (gobject_class, + PROP_INPUT_BLEND_SRC_RGB, + g_param_spec_enum ("blend-src-rgb", "Blend Source RGB", + "Blend factor for source RGB", + GST_TYPE_D3D11_COMPOSITOR_BLEND, + DEFAULT_INPUT_BLEND_SRC_RGB, + G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); + + g_object_class_install_property (gobject_class, + PROP_INPUT_BLEND_SRC_ALPHA, + g_param_spec_enum ("blend-src-alpha", + "Blend Source Alpha", + "Blend factor for source alpha, \"*-color\" values are not allowed", + GST_TYPE_D3D11_COMPOSITOR_BLEND, + DEFAULT_INPUT_BLEND_SRC_ALPHA, + G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); + + g_object_class_install_property (gobject_class, + PROP_INPUT_BLEND_DEST_RGB, + g_param_spec_enum ("blend-dest-rgb", + "Blend Destination RGB", + "Blend factor for destination RGB", + GST_TYPE_D3D11_COMPOSITOR_BLEND, + DEFAULT_INPUT_BLEND_DEST_RGB, + G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); + + g_object_class_install_property (gobject_class, + PROP_INPUT_BLEND_DEST_ALPHA, + g_param_spec_enum ("blend-dest-alpha", + "Blend Destination Alpha", + "Blend factor for destination alpha, " + "\"*-color\" values are not allowed", + GST_TYPE_D3D11_COMPOSITOR_BLEND, + DEFAULT_INPUT_BLEND_DEST_ALPHA, + G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); + + g_object_class_install_property (gobject_class, PROP_INPUT_BLEND_FACTOR_RED, + g_param_spec_float ("blend-factor-red", "Blend Factor Red", + "Blend factor for red component " + "when blend type is \"blend-factor\" or \"inv-blend-factor\"", + 0.0, 1.0, 1.0, + G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); + + g_object_class_install_property (gobject_class, PROP_INPUT_BLEND_FACTOR_GREEN, + g_param_spec_float ("blend-factor-green", "Blend Factor Green", + "Blend factor for green component " + "when blend type is \"blend-factor\" or \"inv-blend-factor\"", + 0.0, 1.0, 1.0, + G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); + + g_object_class_install_property (gobject_class, PROP_INPUT_BLEND_FACTOR_BLUE, + g_param_spec_float ("blend-factor-blue", "Blend Factor Blue", + "Blend factor for blue component " + "when blend type is \"blend-factor\" or \"inv-blend-factor\"", + 0.0, 1.0, 1.0, + G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); + + g_object_class_install_property (gobject_class, PROP_INPUT_BLEND_FACTOR_ALPHA, + g_param_spec_float ("blend-factor-alpha", "Blend Factor Alpha", + "Blend factor for alpha component " + "when blend type is \"blend-factor\" or \"inv-blend-factor\"", + 0.0, 1.0, 1.0, + G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS)); + + pad_class->set_target = + GST_DEBUG_FUNCPTR (gst_d3d11_compositor_bin_input_set_target); +} + +static void +gst_d3d11_compositor_bin_input_init (GstD3D11CompositorBinInput * self) +{ +} + +static void +gst_d3d11_compositor_bin_input_set_property (GObject * object, + guint prop_id, const GValue * value, GParamSpec * pspec) +{ + GstD3D11CompositorBinPad *pad = GST_D3D11_COMPOSITOR_BIN_PAD (object); + + if (pad->target) + g_object_set_property (G_OBJECT (pad->target), pspec->name, value); +} + +static void +gst_d3d11_compositor_bin_input_get_property (GObject * object, + guint prop_id, GValue * value, GParamSpec * pspec) +{ + GstD3D11CompositorBinPad *pad = GST_D3D11_COMPOSITOR_BIN_PAD (object); + + if (pad->target) + g_object_get_property (G_OBJECT (pad->target), pspec->name, value); +} + +static void +gst_d3d11_compositor_bin_input_set_target (GstD3D11CompositorBinPad * pad, + GstPad * target) +{ + GST_D3D11_COMPOSITOR_BIN_PAD_CLASS (input_parent_class)->set_target (pad, + target); + +#define ADD_BINDING(obj,ref,prop) \ + gst_object_add_control_binding (GST_OBJECT (obj), \ + gst_proxy_control_binding_new (GST_OBJECT (obj), prop, \ + GST_OBJECT (ref), prop)); + /* GstVideoAggregatorPad */ + ADD_BINDING (target, pad, "zorder"); + ADD_BINDING (target, pad, "repeat-after-eos"); + /* GstD3D11CompositorPad */ + ADD_BINDING (target, pad, "xpos"); + ADD_BINDING (target, pad, "ypos"); + ADD_BINDING (target, pad, "width"); + ADD_BINDING (target, pad, "height"); + ADD_BINDING (target, pad, "alpha"); + ADD_BINDING (target, pad, "blend-op-rgb"); + ADD_BINDING (target, pad, "blend-op-alpha"); + ADD_BINDING (target, pad, "blend-src-rgb"); + ADD_BINDING (target, pad, "blend-src-alpha"); + ADD_BINDING (target, pad, "blend-dest-rgb"); + ADD_BINDING (target, pad, "blend-dest-alpha"); + ADD_BINDING (target, pad, "blend-factor-red"); + ADD_BINDING (target, pad, "blend-factor-green"); + ADD_BINDING (target, pad, "blend-factor-blue"); + ADD_BINDING (target, pad, "blend-factor-alpha"); +#undef ADD_BINDING +} + +/************************* + * GstD3D11CompositorBin * + *************************/ + +static GstStaticPadTemplate sink_template = GST_STATIC_PAD_TEMPLATE ("sink_%u", + GST_PAD_SINK, + GST_PAD_REQUEST, + GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES + (GST_CAPS_FEATURE_MEMORY_D3D11_MEMORY, GST_D3D11_SINK_FORMATS) ";" + GST_VIDEO_CAPS_MAKE (GST_D3D11_SINK_FORMATS) + )); + +static GstStaticPadTemplate src_template = GST_STATIC_PAD_TEMPLATE ("src", + GST_PAD_SRC, + GST_PAD_ALWAYS, + GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES + (GST_CAPS_FEATURE_MEMORY_D3D11_MEMORY, GST_D3D11_SRC_FORMATS) ";" + GST_VIDEO_CAPS_MAKE (GST_D3D11_SRC_FORMATS) + )); + +enum +{ + PROP_0, + PROP_MIXER, + /* GstAggregator */ + PROP_LATENCY, + PROP_MIN_UPSTREAM_LATENCY, + PROP_START_TIME_SELECTION, + PROP_START_TIME, + PROP_EMIT_SIGNALS, + /* GstD3D11Compositor */ + PROP_ADAPTER, + PROP_BACKGROUND, + PROP_LAST +}; + +/* GstAggregator */ +#define DEFAULT_LATENCY 0 +#define DEFAULT_MIN_UPSTREAM_LATENCY 0 +#define DEFAULT_START_TIME_SELECTION GST_AGGREGATOR_START_TIME_SELECTION_ZERO +#define DEFAULT_START_TIME (-1) +#define DEFAULT_EMIT_SIGNALS FALSE + +/* GstD3D11Compositor */ +#define DEFAULT_ADAPTER -1 +#define DEFAULT_BACKGROUND GST_D3D11_COMPOSITOR_BACKGROUND_CHECKER + +typedef struct _GstD3D11CompositorBinChain +{ + /* without ref */ + GstD3D11CompositorBin *self; + GstD3D11CompositorBinPad *ghost_pad; + GstElement *upload; + GstElement *convert; + + gulong probe_id; +} GstD3D11CompositorBinChain; + +struct _GstD3D11CompositorBin +{ + GstBin parent; + + GstElement *compositor; + + GList *input_chains; + gboolean running; + + gint adapter; +}; + +static void gst_d3d11_compositor_bin_child_proxy_init (gpointer g_iface, + gpointer iface_data); +static void gst_d3d11_compositor_bin_dispose (GObject * object); +static void gst_d3d11_compositor_bin_set_property (GObject * object, + guint prop_id, const GValue * value, GParamSpec * pspec); +static void gst_d3d11_compositor_bin_get_property (GObject * object, + guint prop_id, GValue * value, GParamSpec * pspec); + +static GstStateChangeReturn +gst_d3d11_compositor_bin_change_state (GstElement * element, + GstStateChange transition); +static GstPad *gst_d3d11_compositor_bin_request_new_pad (GstElement * element, + GstPadTemplate * templ, const gchar * name, const GstCaps * caps); +static void gst_d3d11_compositor_bin_release_pad (GstElement * element, + GstPad * pad); + +#define gst_d3d11_compositor_bin_parent_class parent_class +G_DEFINE_TYPE_WITH_CODE (GstD3D11CompositorBin, gst_d3d11_compositor_bin, + GST_TYPE_BIN, G_IMPLEMENT_INTERFACE (GST_TYPE_CHILD_PROXY, + gst_d3d11_compositor_bin_child_proxy_init)); + +static void +gst_d3d11_compositor_bin_class_init (GstD3D11CompositorBinClass * klass) +{ + GObjectClass *gobject_class = G_OBJECT_CLASS (klass); + GstElementClass *element_class = GST_ELEMENT_CLASS (klass); + + gobject_class->dispose = gst_d3d11_compositor_bin_dispose; + gobject_class->set_property = gst_d3d11_compositor_bin_set_property; + gobject_class->get_property = gst_d3d11_compositor_bin_get_property; + + element_class->change_state = + GST_DEBUG_FUNCPTR (gst_d3d11_compositor_bin_change_state); + element_class->request_new_pad = + GST_DEBUG_FUNCPTR (gst_d3d11_compositor_bin_request_new_pad); + element_class->release_pad = + GST_DEBUG_FUNCPTR (gst_d3d11_compositor_bin_release_pad); + + gst_element_class_set_static_metadata (element_class, + "Direct3D11 Compositor Bin", + "Filter/Editor/Video/Compositor", + "Composite multiple video streams via D3D11 API", + "Seungha Yang "); + + gst_element_class_add_static_pad_template_with_gtype (element_class, + &sink_template, GST_TYPE_D3D11_COMPOSITOR_BIN_INPUT); + gst_element_class_add_static_pad_template_with_gtype (element_class, + &src_template, GST_TYPE_D3D11_COMPOSITOR_BIN_PAD); + + g_object_class_install_property (gobject_class, PROP_MIXER, + g_param_spec_object ("mixer", "D3D11 mixer element", + "The d3d11 mixer chain to use", + GST_TYPE_ELEMENT, G_PARAM_READABLE | G_PARAM_STATIC_STRINGS)); + + /*GstAggregator */ + g_object_class_install_property (gobject_class, PROP_LATENCY, + g_param_spec_uint64 ("latency", "Buffer latency", + "Additional latency in live mode to allow upstream " + "to take longer to produce buffers for the current " + "position (in nanoseconds)", 0, G_MAXUINT64, + DEFAULT_LATENCY, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + + g_object_class_install_property (gobject_class, PROP_MIN_UPSTREAM_LATENCY, + g_param_spec_uint64 ("min-upstream-latency", "Buffer latency", + "When sources with a higher latency are expected to be plugged " + "in dynamically after the aggregator has started playing, " + "this allows overriding the minimum latency reported by the " + "initial source(s). This is only taken into account when larger " + "than the actually reported minimum latency. (nanoseconds)", + 0, G_MAXUINT64, + DEFAULT_LATENCY, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + + g_object_class_install_property (gobject_class, PROP_START_TIME_SELECTION, + g_param_spec_enum ("start-time-selection", "Start Time Selection", + "Decides which start time is output", + gst_aggregator_start_time_selection_get_type (), + DEFAULT_START_TIME_SELECTION, + G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + + g_object_class_install_property (gobject_class, PROP_START_TIME, + g_param_spec_uint64 ("start-time", "Start Time", + "Start time to use if start-time-selection=set", 0, + G_MAXUINT64, + DEFAULT_START_TIME, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + + g_object_class_install_property (gobject_class, PROP_EMIT_SIGNALS, + g_param_spec_boolean ("emit-signals", "Emit signals", + "Send signals", DEFAULT_EMIT_SIGNALS, + G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + + /* GstD3D11Compositor */ + g_object_class_install_property (gobject_class, PROP_ADAPTER, + g_param_spec_int ("adapter", "Adapter", + "Adapter index for creating device (-1 for default)", + -1, G_MAXINT32, DEFAULT_ADAPTER, + G_PARAM_READWRITE | GST_PARAM_MUTABLE_READY | + G_PARAM_STATIC_STRINGS)); + + g_object_class_install_property (gobject_class, PROP_BACKGROUND, + g_param_spec_enum ("background", "Background", "Background type", + GST_TYPE_COMPOSITOR_BACKGROUND, + DEFAULT_BACKGROUND, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); +} + +static void +gst_d3d11_compositor_bin_init (GstD3D11CompositorBin * self) +{ + GstPad *pad; + GstPad *gpad; + GstElement *out_convert, *download; + + self->compositor = gst_element_factory_make ("d3d11compositorelement", NULL); + out_convert = gst_element_factory_make ("d3d11colorconvert", NULL); + download = gst_element_factory_make ("d3d11download", NULL); + + gst_bin_add_many (GST_BIN (self), + self->compositor, out_convert, download, NULL); + gst_element_link_many (self->compositor, out_convert, download, NULL); + + gpad = (GstPad *) g_object_new (GST_TYPE_D3D11_COMPOSITOR_BIN_PAD, + "name", "src", "direction", GST_PAD_SRC, NULL); + pad = gst_element_get_static_pad (self->compositor, "src"); + /* GstD3D11CompositorBinPad will hold reference of this compositor srcpad */ + gst_d3d11_compositor_bin_pad_set_target ((GstD3D11CompositorBinPad *) gpad, + pad); + + pad = gst_element_get_static_pad (download, "src"); + gst_ghost_pad_set_target (GST_GHOST_PAD_CAST (gpad), pad); + gst_object_unref (pad); + + gst_element_add_pad (GST_ELEMENT_CAST (self), gpad); +} + +static void +gst_d3d11_compositor_bin_dispose (GObject * object) +{ + GstD3D11CompositorBin *self = GST_D3D11_COMPOSITOR_BIN (object); + GList *iter; + + for (iter = self->input_chains; iter; iter = g_list_next (iter)) { + GstD3D11CompositorBinChain *chain = + (GstD3D11CompositorBinChain *) iter->data; + + if (self->compositor && chain->ghost_pad && chain->ghost_pad->target) { + gst_element_release_request_pad (GST_ELEMENT_CAST (self->compositor), + chain->ghost_pad->target); + gst_d3d11_compositor_bin_pad_unset_target (chain->ghost_pad); + } + } + + if (self->input_chains) + g_list_free_full (self->input_chains, (GDestroyNotify) g_free); + self->input_chains = NULL; + + G_OBJECT_CLASS (parent_class)->dispose (object); +} + +static void +gst_d3d11_compositor_bin_set_property (GObject * object, + guint prop_id, const GValue * value, GParamSpec * pspec) +{ + GstD3D11CompositorBin *self = GST_D3D11_COMPOSITOR_BIN (object); + + switch (prop_id) { + case PROP_ADAPTER: + self->adapter = g_value_get_int (value); + /* fallthrough */ + default: + g_object_set_property (G_OBJECT (self->compositor), pspec->name, value); + break; + } +} + +static void +gst_d3d11_compositor_bin_get_property (GObject * object, + guint prop_id, GValue * value, GParamSpec * pspec) +{ + GstD3D11CompositorBin *self = GST_D3D11_COMPOSITOR_BIN (object); + + switch (prop_id) { + case PROP_MIXER: + g_value_set_object (value, self->compositor); + break; + case PROP_ADAPTER: + g_value_set_int (value, self->adapter); + break; + default: + g_object_get_property (G_OBJECT (self->compositor), pspec->name, value); + break; + } +} + +static GstStateChangeReturn +gst_d3d11_compositor_bin_change_state (GstElement * element, + GstStateChange transition) +{ + GstD3D11CompositorBin *self = GST_D3D11_COMPOSITOR_BIN (element); + GstStateChangeReturn ret; + + switch (transition) { + case GST_STATE_CHANGE_NULL_TO_READY: + GST_OBJECT_LOCK (element); + self->running = TRUE; + GST_OBJECT_UNLOCK (element); + break; + default: + break; + } + + ret = GST_ELEMENT_CLASS (parent_class)->change_state (element, transition); + if (ret == GST_STATE_CHANGE_FAILURE) + return ret; + + switch (transition) { + case GST_STATE_CHANGE_READY_TO_NULL: + GST_OBJECT_LOCK (self); + self->running = FALSE; + GST_OBJECT_UNLOCK (self); + default: + break; + } + + return ret; +} + +static GstD3D11CompositorBinChain * +gst_d3d11_compositor_bin_input_chain_new (GstD3D11CompositorBin * self, + GstPad * compositor_pad) +{ + GstD3D11CompositorBinChain *chain; + GstPad *pad; + + chain = g_new0 (GstD3D11CompositorBinChain, 1); + + chain->self = self; + + chain->upload = gst_element_factory_make ("d3d11upload", NULL); + chain->convert = gst_element_factory_make ("d3d11colorconvert", NULL); + + /* 1. Create child elements and like */ + gst_bin_add_many (GST_BIN (self), chain->upload, chain->convert, NULL); + + gst_element_link (chain->upload, chain->convert); + pad = gst_element_get_static_pad (chain->convert, "src"); + gst_pad_link (pad, compositor_pad); + gst_object_unref (pad); + + chain->ghost_pad = (GstD3D11CompositorBinPad *) + g_object_new (GST_TYPE_D3D11_COMPOSITOR_BIN_INPUT, "name", + GST_OBJECT_NAME (compositor_pad), "direction", GST_PAD_SINK, NULL); + + /* transfer ownership of compositor pad */ + gst_d3d11_compositor_bin_pad_set_target (chain->ghost_pad, compositor_pad); + + pad = gst_element_get_static_pad (chain->upload, "sink"); + gst_ghost_pad_set_target (GST_GHOST_PAD_CAST (chain->ghost_pad), pad); + gst_object_unref (pad); + + GST_OBJECT_LOCK (self); + if (self->running) + gst_pad_set_active (GST_PAD (chain->ghost_pad), TRUE); + GST_OBJECT_UNLOCK (self); + + gst_element_add_pad (GST_ELEMENT_CAST (self), + GST_PAD_CAST (chain->ghost_pad)); + + gst_element_sync_state_with_parent (chain->upload); + gst_element_sync_state_with_parent (chain->convert); + + return chain; +} + +static void +gst_d3d11_compositor_bin_input_chain_free (GstD3D11CompositorBinChain * chain) +{ + if (!chain) + return; + + if (chain->ghost_pad && chain->probe_id) { + gst_pad_remove_probe (GST_PAD_CAST (chain->ghost_pad), chain->probe_id); + chain->probe_id = 0; + } + + if (chain->upload) { + gst_element_set_state (chain->upload, GST_STATE_NULL); + gst_bin_remove (GST_BIN_CAST (chain->self), chain->upload); + } + + if (chain->convert) { + gst_element_set_state (chain->convert, GST_STATE_NULL); + gst_bin_remove (GST_BIN_CAST (chain->self), chain->convert); + } + + if (chain->ghost_pad && chain->ghost_pad->target) { + gst_element_release_request_pad (chain->self->compositor, + chain->ghost_pad->target); + gst_d3d11_compositor_bin_pad_unset_target (chain->ghost_pad); + } + + g_free (chain); +} + +static GstPad * +gst_d3d11_compositor_bin_request_new_pad (GstElement * element, + GstPadTemplate * templ, const gchar * name, const GstCaps * caps) +{ + GstD3D11CompositorBin *self = GST_D3D11_COMPOSITOR_BIN (element); + GstElementClass *compositor_class = GST_ELEMENT_GET_CLASS (self->compositor); + GstPad *compositor_pad; + GstD3D11CompositorBinChain *chain; + GstPadTemplate *compositor_templ = NULL; + GList *templ_list; + GList *iter; + + templ_list = gst_element_class_get_pad_template_list (compositor_class); + for (iter = templ_list; iter; iter = g_list_next (iter)) { + GstPadTemplate *t = (GstPadTemplate *) iter->data; + if (GST_PAD_TEMPLATE_DIRECTION (t) != GST_PAD_SINK || + GST_PAD_TEMPLATE_PRESENCE (t) != GST_PAD_REQUEST) + continue; + + compositor_templ = t; + break; + } + + g_assert (compositor_templ); + + compositor_pad = + gst_element_request_pad (self->compositor, compositor_templ, name, caps); + if (!compositor_pad) { + GST_WARNING_OBJECT (self, "Failed to request pad"); + return NULL; + } + + chain = gst_d3d11_compositor_bin_input_chain_new (self, compositor_pad); + g_assert (chain); + + GST_OBJECT_LOCK (self); + self->input_chains = g_list_append (self->input_chains, chain); + GST_OBJECT_UNLOCK (self); + + gst_child_proxy_child_added (GST_CHILD_PROXY (self), + G_OBJECT (chain->ghost_pad), GST_OBJECT_NAME (chain->ghost_pad)); + + GST_DEBUG_OBJECT (element, "Created new pad %s:%s", + GST_DEBUG_PAD_NAME (chain->ghost_pad)); + + return GST_PAD_CAST (chain->ghost_pad); +} + +static void +gst_d3d11_compositor_bin_release_pad (GstElement * element, GstPad * pad) +{ + GstD3D11CompositorBin *self = GST_D3D11_COMPOSITOR_BIN (element); + GList *iter; + gboolean found = FALSE; + + GST_DEBUG_OBJECT (self, "Releasing pad %s:%s", GST_DEBUG_PAD_NAME (pad)); + + GST_OBJECT_LOCK (self); + for (iter = self->input_chains; iter; iter = g_list_next (iter)) { + GstD3D11CompositorBinChain *chain = + (GstD3D11CompositorBinChain *) iter->data; + + if (pad == GST_PAD_CAST (chain->ghost_pad)) { + self->input_chains = g_list_delete_link (self->input_chains, iter); + GST_OBJECT_UNLOCK (self); + + gst_d3d11_compositor_bin_input_chain_free (chain); + found = TRUE; + break; + } + } + + if (!found) { + GST_OBJECT_UNLOCK (self); + GST_WARNING_OBJECT (self, "Unknown pad to release %s:%s", + GST_DEBUG_PAD_NAME (pad)); + } + + gst_element_remove_pad (element, pad); +} + +static GObject * +gst_d3d11_compositor_bin_child_proxy_get_child_by_index (GstChildProxy * proxy, + guint index) +{ + GstD3D11CompositorBin *self = GST_D3D11_COMPOSITOR_BIN (proxy); + GstBin *bin = GST_BIN_CAST (proxy); + GObject *res = NULL; + + GST_OBJECT_LOCK (self); + /* XXX: not exactly thread safe with ordering */ + if (index < bin->numchildren) { + if ((res = g_list_nth_data (bin->children, index))) + gst_object_ref (res); + } else { + GstD3D11CompositorBinChain *chain; + if ((chain = + g_list_nth_data (self->input_chains, index - bin->numchildren))) { + res = gst_object_ref (chain->ghost_pad); + } + } + GST_OBJECT_UNLOCK (self); + + return res; +} + +static guint +gst_d3d11_compositor_bin_child_proxy_get_children_count (GstChildProxy * proxy) +{ + GstD3D11CompositorBin *self = GST_D3D11_COMPOSITOR_BIN (proxy); + guint count = 0; + + GST_OBJECT_LOCK (self); + count = GST_BIN_CAST (self)->numchildren + g_list_length (self->input_chains); + GST_OBJECT_UNLOCK (self); + GST_INFO_OBJECT (self, "Children Count: %d", count); + + return count; +} + +static void +gst_d3d11_compositor_bin_child_proxy_init (gpointer g_iface, + gpointer iface_data) +{ + GstChildProxyInterface *iface = g_iface; + + iface->get_child_by_index = + gst_d3d11_compositor_bin_child_proxy_get_child_by_index; + iface->get_children_count = + gst_d3d11_compositor_bin_child_proxy_get_children_count; +} diff --git a/sys/d3d11/gstd3d11compositorbin.h b/sys/d3d11/gstd3d11compositorbin.h new file mode 100644 index 0000000000..7b8a58d45d --- /dev/null +++ b/sys/d3d11/gstd3d11compositorbin.h @@ -0,0 +1,65 @@ +/* + * GStreamer + * Copyright (C) 2020 Seungha Yang + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifndef __GST_D3D11_COMPOSITOR_BIN_H__ +#define __GST_D3D11_COMPOSITOR_BIN_H__ + +#include +#include +#include + +G_BEGIN_DECLS + +#define GST_TYPE_D3D11_COMPOSITOR_BIN_PAD (gst_d3d11_compositor_bin_pad_get_type()) +#define GST_D3D11_COMPOSITOR_BIN_PAD(obj) \ + (G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_D3D11_COMPOSITOR_BIN_PAD, GstD3D11CompositorBinPad)) +#define GST_D3D11_COMPOSITOR_BIN_PAD_CLASS(klass) \ + (G_TYPE_CHECK_CLASS_CAST((klass),GST_TYPE_D3D11_COMPOSITOR_BIN_PAD, GstD3D11CompositorBinPadClass)) +#define GST_IS_D3D11_COMPOSITOR_BIN_PAD(obj) \ + (G_TYPE_CHECK_INSTANCE_TYPE((obj),GST_TYPE_D3D11_COMPOSITOR_BIN_PAD)) +#define GST_IS_D3D11_COMPOSITOR_BIN_PAD_CLASS(klass) \ + (G_TYPE_CHECK_CLASS_TYPE((klass),GST_TYPE_D3D11_COMPOSITOR_BIN_PAD)) +#define GST_D3D11_COMPOSITOR_BIN_PAD_GET_CLASS(obj) \ + (G_TYPE_INSTANCE_GET_CLASS((obj),GST_TYPE_D3D11_COMPOSITOR_BIN_PAD,GstD3D11CompositorBinPadClass)) + +typedef struct _GstD3D11CompositorBinPad GstD3D11CompositorBinPad; +typedef struct _GstD3D11CompositorBinPadClass GstD3D11CompositorBinPadClass; + +struct _GstD3D11CompositorBinPadClass +{ + GstGhostPadClass parent_class; + + void (*set_target) (GstD3D11CompositorBinPad * pad, GstPad * target); +}; + +GType gst_d3d11_compositor_bin_pad_get_type (void); +G_DEFINE_AUTOPTR_CLEANUP_FUNC(GstD3D11CompositorBinPad, gst_object_unref) + +#define GST_TYPE_D3D11_COMPOSITOR_BIN_INPUT (gst_d3d11_compositor_bin_input_get_type()) +G_DECLARE_FINAL_TYPE (GstD3D11CompositorBinInput, gst_d3d11_compositor_bin_input, + GST, D3D11_COMPOSITOR_BIN_INPUT, GstD3D11CompositorBinPad); + +#define GST_TYPE_D3D11_COMPOSITOR_BIN (gst_d3d11_compositor_bin_get_type()) +G_DECLARE_FINAL_TYPE (GstD3D11CompositorBin, gst_d3d11_compositor_bin, + GST, D3D11_COMPOSITOR_BIN, GstBin) + +G_END_DECLS + +#endif /* __GST_D3D11_COMPOSITOR_BIN_H__ */ diff --git a/sys/d3d11/meson.build b/sys/d3d11/meson.build index 478c2f6056..c88a4a6a78 100644 --- a/sys/d3d11/meson.build +++ b/sys/d3d11/meson.build @@ -16,6 +16,8 @@ d3d11_sources = [ 'gstd3d11colorconverter.c', 'gstd3d11overlaycompositor.c', 'gstd3d11videoprocessor.c', + 'gstd3d11compositor.c', + 'gstd3d11compositorbin.c', ] d3d11_dec_sources = [ @@ -192,7 +194,7 @@ gstd3d11 = library('gstd3d11', c_args : gst_plugins_bad_args + extra_c_args, cpp_args: gst_plugins_bad_args, include_directories : [configinc], - dependencies : [gstbase_dep, gstvideo_dep, gmodule_dep, d3d11_lib, dxgi_lib] + extra_dep, + dependencies : [gstbase_dep, gstvideo_dep, gmodule_dep, gstcontroller_dep, d3d11_lib, dxgi_lib] + extra_dep, install : true, install_dir : plugins_install_dir, ) diff --git a/sys/d3d11/plugin.c b/sys/d3d11/plugin.c index 3dfddc87fd..9dbdc1a67d 100644 --- a/sys/d3d11/plugin.c +++ b/sys/d3d11/plugin.c @@ -30,6 +30,8 @@ #include "gstd3d11colorconvert.h" #include "gstd3d11videosinkbin.h" #include "gstd3d11shader.h" +#include "gstd3d11compositor.h" +#include "gstd3d11compositorbin.h" #ifdef HAVE_DXVA_H #include "gstd3d11utils.h" #include "gstd3d11h264dec.h" @@ -47,6 +49,7 @@ GST_DEBUG_CATEGORY (gst_d3d11_device_debug); GST_DEBUG_CATEGORY (gst_d3d11_overlay_compositor_debug); GST_DEBUG_CATEGORY (gst_d3d11_window_debug); GST_DEBUG_CATEGORY (gst_d3d11_video_processor_debug); +GST_DEBUG_CATEGORY (gst_d3d11_compositor_debug); #if (HAVE_D3D11SDKLAYERS_H || HAVE_DXGIDEBUG_H) GST_DEBUG_CATEGORY (gst_d3d11_debug_layer_debug); @@ -90,6 +93,8 @@ plugin_init (GstPlugin * plugin) "d3d11window", 0, "d3d11window"); GST_DEBUG_CATEGORY_INIT (gst_d3d11_video_processor_debug, "d3d11videoprocessor", 0, "d3d11videoprocessor"); + GST_DEBUG_CATEGORY_INIT (gst_d3d11_compositor_debug, + "d3d11compositor", 0, "d3d11compositor element"); #if (HAVE_D3D11SDKLAYERS_H || HAVE_DXGIDEBUG_H) /* NOTE: enabled only for debug build */ @@ -137,6 +142,11 @@ plugin_init (GstPlugin * plugin) gst_element_register (plugin, "d3d11videosink", video_sink_rank, GST_TYPE_D3D11_VIDEO_SINK_BIN); + gst_element_register (plugin, + "d3d11compositorelement", GST_RANK_NONE, GST_TYPE_D3D11_COMPOSITOR); + gst_element_register (plugin, + "d3d11compositor", GST_RANK_SECONDARY, GST_TYPE_D3D11_COMPOSITOR_BIN); + #ifdef HAVE_DXVA_H /* DXVA2 API is availble since Windows 8 */ if (gst_d3d11_is_windows_8_or_greater ()) {