From 63bb0b8de7fb7c206b2d9a598226df401f3863e0 Mon Sep 17 00:00:00 2001 From: Seungha Yang Date: Sun, 22 Oct 2023 01:10:54 +0900 Subject: [PATCH] d3d11videosink: Add support for transform and MSAA Adding properties for 3D rotation with arbitrary angle and scaling. And adding Multi Sampling Anti-Aliasing rendering support to smooth borders if arbitrary angle is applied Part-of: --- .../docs/plugins/gst_plugins_cache.json | 133 +++++++++ .../sys/d3d11/gstd3d11pluginutils.cpp | 24 ++ .../sys/d3d11/gstd3d11pluginutils.h | 11 + .../sys/d3d11/gstd3d11videosink.cpp | 203 ++++++++++++- .../sys/d3d11/gstd3d11window.cpp | 280 ++++++++++++++++-- .../sys/d3d11/gstd3d11window.h | 21 +- .../gst-plugins-bad/sys/d3d11/meson.build | 25 ++ 7 files changed, 673 insertions(+), 24 deletions(-) diff --git a/subprojects/gst-plugins-bad/docs/plugins/gst_plugins_cache.json b/subprojects/gst-plugins-bad/docs/plugins/gst_plugins_cache.json index 8da20133ec..a6e871f0f7 100644 --- a/subprojects/gst-plugins-bad/docs/plugins/gst_plugins_cache.json +++ b/subprojects/gst-plugins-bad/docs/plugins/gst_plugins_cache.json @@ -10212,6 +10212,20 @@ "type": "gboolean", "writable": true }, + "fov": { + "blurb": "Field of view angle in degrees", + "conditionally-available": false, + "construct": false, + "construct-only": false, + "controllable": false, + "default": "90", + "max": "3.40282e+38", + "min": "0", + "mutable": "null", + "readable": true, + "type": "gfloat", + "writable": true + }, "fullscreen": { "blurb": "Ignored when \"fullscreen-toggle-mode\" does not include \"property\"", "conditionally-available": false, @@ -10248,6 +10262,30 @@ "type": "GstVideoGammaMode", "writable": true }, + "msaa": { + "blurb": "MSAA (Multi-Sampling Anti-Aliasing) level", + "conditionally-available": false, + "construct": false, + "construct-only": false, + "controllable": false, + "default": "disabled (0)", + "mutable": "null", + "readable": true, + "type": "GstD3D11MSAAMode", + "writable": true + }, + "ortho": { + "blurb": "Use orthographic projection", + "conditionally-available": false, + "construct": false, + "construct-only": false, + "controllable": false, + "default": "false", + "mutable": "null", + "readable": true, + "type": "gboolean", + "writable": true + }, "primaries-mode": { "blurb": "Primaries conversion mode", "conditionally-available": false, @@ -10282,6 +10320,76 @@ "readable": true, "type": "GstVideoOrientationMethod", "writable": true + }, + "rotation-x": { + "blurb": "x-axis rotation angle in degrees", + "conditionally-available": false, + "construct": false, + "construct-only": false, + "controllable": false, + "default": "0", + "max": "3.40282e+38", + "min": "-3.40282e+38", + "mutable": "null", + "readable": true, + "type": "gfloat", + "writable": true + }, + "rotation-y": { + "blurb": "y-axis rotation angle in degrees", + "conditionally-available": false, + "construct": false, + "construct-only": false, + "controllable": false, + "default": "0", + "max": "3.40282e+38", + "min": "-3.40282e+38", + "mutable": "null", + "readable": true, + "type": "gfloat", + "writable": true + }, + "rotation-z": { + "blurb": "z-axis rotation angle in degrees", + "conditionally-available": false, + "construct": false, + "construct-only": false, + "controllable": false, + "default": "0", + "max": "3.40282e+38", + "min": "-3.40282e+38", + "mutable": "null", + "readable": true, + "type": "gfloat", + "writable": true + }, + "scale-x": { + "blurb": "Scale multiplier for x-axis", + "conditionally-available": false, + "construct": false, + "construct-only": false, + "controllable": false, + "default": "1", + "max": "3.40282e+38", + "min": "-3.40282e+38", + "mutable": "null", + "readable": true, + "type": "gfloat", + "writable": true + }, + "scale-y": { + "blurb": "Scale multiplier for y-axis", + "conditionally-available": false, + "construct": false, + "construct-only": false, + "controllable": false, + "default": "1", + "max": "3.40282e+38", + "min": "-3.40282e+38", + "mutable": "null", + "readable": true, + "type": "gfloat", + "writable": true } }, "rank": "primary", @@ -10778,6 +10886,31 @@ } ] }, + "GstD3D11MSAAMode": { + "kind": "enum", + "values": [ + { + "desc": "Disabled", + "name": "disabled", + "value": "0" + }, + { + "desc": "2x MSAA", + "name": "2x", + "value": "1" + }, + { + "desc": "4x MSAA", + "name": "4x", + "value": "2" + }, + { + "desc": "8x MSAA", + "name": "8x", + "value": "3" + } + ] + }, "GstD3D11SamplingMethod": { "kind": "enum", "values": [ diff --git a/subprojects/gst-plugins-bad/sys/d3d11/gstd3d11pluginutils.cpp b/subprojects/gst-plugins-bad/sys/d3d11/gstd3d11pluginutils.cpp index 23d32bee23..9718dd961a 100644 --- a/subprojects/gst-plugins-bad/sys/d3d11/gstd3d11pluginutils.cpp +++ b/subprojects/gst-plugins-bad/sys/d3d11/gstd3d11pluginutils.cpp @@ -77,6 +77,30 @@ gst_d3d11_alpha_mode_get_type (void) return type; } +/** + * GstD3D11MSAAMode: + * + * Since: 1.24 + */ +GType +gst_d3d11_msaa_mode_get_type (void) +{ + static GType type = 0; + static const GEnumValue msaa_mode[] = { + {GST_D3D11_MSAA_DISABLED, "Disabled", "disabled"}, + {GST_D3D11_MSAA_2X, "2x MSAA", "2x"}, + {GST_D3D11_MSAA_4X, "4x MSAA", "4x"}, + {GST_D3D11_MSAA_8X, "8x MSAA", "8x"}, + {0, nullptr, nullptr}, + }; + + GST_D3D11_CALL_ONCE_BEGIN { + type = g_enum_register_static ("GstD3D11MSAAMode", msaa_mode); + } GST_D3D11_CALL_ONCE_END; + + return type; +} + /* Max Texture Dimension for feature level 11_0 ~ 12_1 */ static guint _gst_d3d11_texture_max_dimension = 16384; diff --git a/subprojects/gst-plugins-bad/sys/d3d11/gstd3d11pluginutils.h b/subprojects/gst-plugins-bad/sys/d3d11/gstd3d11pluginutils.h index e7020146c9..a2fd025963 100644 --- a/subprojects/gst-plugins-bad/sys/d3d11/gstd3d11pluginutils.h +++ b/subprojects/gst-plugins-bad/sys/d3d11/gstd3d11pluginutils.h @@ -50,6 +50,17 @@ typedef enum #define GST_TYPE_D3D11_ALPHA_MODE (gst_d3d11_alpha_mode_get_type()) GType gst_d3d11_alpha_mode_get_type (void); +typedef enum +{ + GST_D3D11_MSAA_DISABLED, + GST_D3D11_MSAA_2X, + GST_D3D11_MSAA_4X, + GST_D3D11_MSAA_8X, +} GstD3D11MSAAMode; + +#define GST_TYPE_D3D11_MSAA_MODE (gst_d3d11_msaa_mode_get_type()) +GType gst_d3d11_msaa_mode_get_type (void); + void gst_d3d11_plugin_utils_init (D3D_FEATURE_LEVEL feature_level); GstCaps * gst_d3d11_get_updated_template_caps (GstStaticCaps * template_caps); diff --git a/subprojects/gst-plugins-bad/sys/d3d11/gstd3d11videosink.cpp b/subprojects/gst-plugins-bad/sys/d3d11/gstd3d11videosink.cpp index 490d12adfb..b819fec899 100644 --- a/subprojects/gst-plugins-bad/sys/d3d11/gstd3d11videosink.cpp +++ b/subprojects/gst-plugins-bad/sys/d3d11/gstd3d11videosink.cpp @@ -65,6 +65,14 @@ enum PROP_PRIMARIES_MODE, PROP_DISPLAY_FORMAT, PROP_EMIT_PRESENT, + PROP_FOV, + PROP_ORTHO, + PROP_ROTATION_X, + PROP_ROTATION_Y, + PROP_ROTATION_Z, + PROP_SCALE_X, + PROP_SCALE_Y, + PROP_MSAA, PROP_RENDER_RECTANGE, }; @@ -78,6 +86,11 @@ enum #define DEFAULT_PRIMARIES_MODE GST_VIDEO_PRIMARIES_MODE_NONE #define DEFAULT_DISPLAY_FORMAT DXGI_FORMAT_UNKNOWN #define DEFAULT_EMIT_PRESENT FALSE +#define DEFAULT_ROTATION 0.0f +#define DEFAULT_SCALE 1.0f +#define DEFAULT_FOV 90.0f +#define DEFAULT_ORTHO FALSE +#define DEFAULT_MSAA GST_D3D11_MSAA_DISABLED /** * GstD3D11VideoSinkDisplayFormat: @@ -188,6 +201,14 @@ struct _GstD3D11VideoSink GstVideoPrimariesMode primaries_mode; DXGI_FORMAT display_format; gboolean emit_present; + gfloat fov; + gboolean ortho; + gfloat rotation_x; + gfloat rotation_y; + gfloat rotation_z; + gfloat scale_x; + gfloat scale_y; + GstD3D11MSAAMode msaa; /* saved render rectangle until we have a window */ GstVideoRectangle render_rect; @@ -399,6 +420,110 @@ gst_d3d11_video_sink_class_init (GstD3D11VideoSinkClass * klass) (GParamFlags) (G_PARAM_READWRITE | GST_PARAM_MUTABLE_READY | G_PARAM_STATIC_STRINGS))); + /** + * GstD3D11VideoSink:fov: + * + * Field of view angle in degrees + * + * Since: 1.24 + */ + g_object_class_install_property (gobject_class, PROP_FOV, + g_param_spec_float ("fov", "Fov", + "Field of view angle in degrees", + 0, G_MAXFLOAT, DEFAULT_FOV, + (GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS))); + + /** + * GstD3D11VideoSink:ortho: + * + * Use orthographic projection + * + * Since: 1.24 + */ + g_object_class_install_property (gobject_class, PROP_ORTHO, + g_param_spec_boolean ("ortho", "Orthographic", + "Use orthographic projection", DEFAULT_ORTHO, + (GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS))); + + /** + * GstD3D11VideoSink:rotation-x: + * + * x-axis rotation angle to be applied prior to "rotate-method" + * + * Since: 1.24 + */ + g_object_class_install_property (gobject_class, PROP_ROTATION_X, + g_param_spec_float ("rotation-x", "Rotation X", + "x-axis rotation angle in degrees", + -G_MAXFLOAT, G_MAXFLOAT, DEFAULT_ROTATION, + (GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS))); + + /** + * GstD3D11VideoSink:rotation-y: + * + * y-axis rotation angle to be applied prior to "rotate-method" + * + * Since: 1.24 + */ + g_object_class_install_property (gobject_class, PROP_ROTATION_Y, + g_param_spec_float ("rotation-y", "Rotation Y", + "y-axis rotation angle in degrees", + -G_MAXFLOAT, G_MAXFLOAT, DEFAULT_ROTATION, + (GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS))); + + /** + * GstD3D11VideoSink:rotation-z: + * + * z-axis rotation angle to be applied prior to "rotate-method" + * + * Since: 1.24 + */ + g_object_class_install_property (gobject_class, PROP_ROTATION_Z, + g_param_spec_float ("rotation-z", "Rotation Z", + "z-axis rotation angle in degrees", + -G_MAXFLOAT, G_MAXFLOAT, DEFAULT_ROTATION, + (GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS))); + + /** + * GstD3D11VideoSink:scale-x: + * + * Scale multiplier for x-axis + * + * Since: 1.24 + */ + g_object_class_install_property (gobject_class, PROP_SCALE_X, + g_param_spec_float ("scale-x", "Scale X", + "Scale multiplier for x-axis", + -G_MAXFLOAT, G_MAXFLOAT, DEFAULT_SCALE, + (GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS))); + + /** + * GstD3D11VideoSink:scale-y: + * + * Scale multiplier for y-axis + * + * Since: 1.24 + */ + g_object_class_install_property (gobject_class, PROP_SCALE_Y, + g_param_spec_float ("scale-y", "Scale Y", + "Scale multiplier for y-axis", + -G_MAXFLOAT, G_MAXFLOAT, DEFAULT_SCALE, + (GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS))); + + /** + * GstD3D11VideoSink:msaa: + * + * MSAA (Multi-Sampling Anti-Aliasing) level + * + * Since: 1.24 + */ + g_object_class_install_property (gobject_class, PROP_MSAA, + g_param_spec_enum ("msaa", "MSAA", + "MSAA (Multi-Sampling Anti-Aliasing) level", + GST_TYPE_D3D11_MSAA_MODE, DEFAULT_MSAA, + (GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS))); + + /** * GstD3D11VideoSink:render-rectangle: * @@ -503,6 +628,7 @@ gst_d3d11_video_sink_class_init (GstD3D11VideoSinkClass * klass) (GstPluginAPIFlags) 0); gst_type_mark_as_plugin_api (GST_TYPE_D3D11_VIDEO_SINK_DISPLAY_FORMAT, (GstPluginAPIFlags) 0); + gst_type_mark_as_plugin_api (GST_TYPE_D3D11_MSAA_MODE, (GstPluginAPIFlags) 0); } static void @@ -518,6 +644,14 @@ gst_d3d11_video_sink_init (GstD3D11VideoSink * self) self->primaries_mode = DEFAULT_PRIMARIES_MODE; self->display_format = DEFAULT_DISPLAY_FORMAT; self->emit_present = DEFAULT_EMIT_PRESENT; + self->fov = DEFAULT_FOV; + self->ortho = DEFAULT_ORTHO; + self->rotation_x = DEFAULT_ROTATION; + self->rotation_y = DEFAULT_ROTATION; + self->rotation_z = DEFAULT_ROTATION; + self->scale_x = DEFAULT_SCALE; + self->scale_y = DEFAULT_SCALE; + self->msaa = DEFAULT_MSAA; InitializeCriticalSection (&self->lock); } @@ -579,6 +713,39 @@ gst_d3d11_videosink_set_property (GObject * object, guint prop_id, case PROP_EMIT_PRESENT: self->emit_present = g_value_get_boolean (value); break; + case PROP_FOV: + self->fov = g_value_get_float (value); + gst_d3d11_video_sink_set_orientation (self, self->method, FALSE); + break; + case PROP_ORTHO: + self->ortho = g_value_get_boolean (value); + gst_d3d11_video_sink_set_orientation (self, self->method, FALSE); + break; + case PROP_ROTATION_X: + self->rotation_x = g_value_get_float (value); + gst_d3d11_video_sink_set_orientation (self, self->method, FALSE); + break; + case PROP_ROTATION_Y: + self->rotation_y = g_value_get_float (value); + gst_d3d11_video_sink_set_orientation (self, self->method, FALSE); + break; + case PROP_ROTATION_Z: + self->rotation_z = g_value_get_float (value); + gst_d3d11_video_sink_set_orientation (self, self->method, FALSE); + break; + case PROP_SCALE_X: + self->scale_x = g_value_get_float (value); + gst_d3d11_video_sink_set_orientation (self, self->method, FALSE); + break; + case PROP_SCALE_Y: + self->scale_y = g_value_get_float (value); + gst_d3d11_video_sink_set_orientation (self, self->method, FALSE); + break; + case PROP_MSAA: + self->msaa = (GstD3D11MSAAMode) g_value_get_enum (value); + if (self->window) + gst_d3d11_window_set_msaa_mode (self->window, self->msaa); + break; case PROP_RENDER_RECTANGE: gst_video_overlay_set_property (object, PROP_RENDER_RECTANGE, PROP_RENDER_RECTANGE, value); @@ -634,6 +801,30 @@ gst_d3d11_videosink_get_property (GObject * object, guint prop_id, case PROP_EMIT_PRESENT: g_value_set_boolean (value, self->emit_present); break; + case PROP_FOV: + g_value_set_float (value, self->fov); + break; + case PROP_ORTHO: + g_value_set_boolean (value, self->ortho); + break; + case PROP_ROTATION_X: + g_value_set_float (value, self->rotation_x); + break; + case PROP_ROTATION_Y: + g_value_set_float (value, self->rotation_x); + break; + case PROP_ROTATION_Z: + g_value_set_float (value, self->rotation_z); + break; + case PROP_SCALE_X: + g_value_set_float (value, self->scale_x); + break; + case PROP_SCALE_Y: + g_value_set_float (value, self->scale_y); + break; + case PROP_MSAA: + g_value_set_enum (value, self->msaa); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); break; @@ -1056,7 +1247,10 @@ done: "enable-navigation-events", self->enable_navigation_events, "emit-present", self->emit_present, nullptr); - gst_d3d11_window_set_orientation (self->window, self->selected_method); + gst_d3d11_window_set_orientation (self->window, self->selected_method, + self->fov, self->ortho, self->rotation_x, self->rotation_y, + self->rotation_z, self->scale_x, self->scale_y); + gst_d3d11_window_set_msaa_mode (self->window, self->msaa); g_signal_connect (self->window, "key-event", G_CALLBACK (gst_d3d11_video_sink_key_event), self); @@ -1320,8 +1514,11 @@ gst_d3d11_video_sink_set_orientation (GstD3D11VideoSink * self, self->selected_method = self->method; } - if (self->window) - gst_d3d11_window_set_orientation (self->window, self->selected_method); + if (self->window) { + gst_d3d11_window_set_orientation (self->window, self->selected_method, + self->fov, self->ortho, self->rotation_x, self->rotation_y, + self->rotation_z, self->scale_x, self->scale_y); + } } static void diff --git a/subprojects/gst-plugins-bad/sys/d3d11/gstd3d11window.cpp b/subprojects/gst-plugins-bad/sys/d3d11/gstd3d11window.cpp index ff0aee0fda..551e794237 100644 --- a/subprojects/gst-plugins-bad/sys/d3d11/gstd3d11window.cpp +++ b/subprojects/gst-plugins-bad/sys/d3d11/gstd3d11window.cpp @@ -36,8 +36,13 @@ #include +/* Disable platform-specific intrinsics */ +#define _XM_NO_INTRINSICS_ +#include + /* *INDENT-OFF* */ using namespace Microsoft::WRL; +using namespace DirectX; /* *INDENT-ON* */ GST_DEBUG_CATEGORY_EXTERN (gst_d3d11_window_debug); @@ -105,7 +110,7 @@ static void gst_d3d11_window_get_property (GObject * object, guint prop_id, GValue * value, GParamSpec * pspec); static void gst_d3d11_window_dispose (GObject * object); static GstFlowReturn gst_d3d11_window_present (GstD3D11Window * self, - GstBuffer * buffer, GstBuffer * render_target); + GstBuffer * buffer, GstBuffer * render_target, GstBuffer * multisample); static void gst_d3d11_window_on_resize_default (GstD3D11Window * window, guint width, guint height); static GstFlowReturn gst_d3d11_window_prepare_default (GstD3D11Window * window, @@ -194,6 +199,14 @@ gst_d3d11_window_init (GstD3D11Window * self) self->fullscreen_toggle_mode = GST_D3D11_WINDOW_FULLSCREEN_TOGGLE_MODE_NONE; self->fullscreen = DEFAULT_FULLSCREEN; self->emit_present = DEFAULT_EMIT_PRESENT; + self->fov = 90.0f; + self->ortho = FALSE; + self->rotation_x = 0.0f; + self->rotation_y = 0.0f; + self->rotation_z = 0.0f; + self->scale_x = 1.0f; + self->scale_y = 1.0f; + self->msaa = GST_D3D11_MSAA_DISABLED; } static void @@ -279,6 +292,7 @@ gst_d3d11_window_dispose (GObject * object) gst_clear_object (&self->compositor); gst_clear_object (&self->converter); + gst_clear_buffer (&self->msaa_buffer); gst_clear_buffer (&self->cached_buffer); gst_clear_object (&self->device); @@ -297,14 +311,19 @@ gst_d3d11_window_on_resize_default (GstD3D11Window * self, guint width, GstVideoRectangle src_rect, dst_rect, rst_rect; IDXGISwapChain *swap_chain; GstMemory *mem; + GstMemory *msaa_mem = nullptr; GstD3D11Memory *dmem; ID3D11RenderTargetView *rtv; ID3D11DeviceContext *context; + ID3D11Device *device_handle; gsize size; GstD3D11DeviceLockGuard lk (device); const FLOAT clear_color[] = { 0.0, 0.0, 0.0, 1.0 }; + UINT quality_levels = 0; + UINT sample_count = 1; gst_clear_buffer (&self->backbuffer); + gst_clear_buffer (&self->msaa_buffer); if (!self->swap_chain) return; @@ -350,12 +369,56 @@ gst_d3d11_window_on_resize_default (GstD3D11Window * self, guint width, return; } + switch (self->msaa) { + case GST_D3D11_MSAA_2X: + sample_count = 2; + break; + case GST_D3D11_MSAA_4X: + sample_count = 4; + break; + case GST_D3D11_MSAA_8X: + sample_count = 8; + break; + default: + break; + } + + device_handle = gst_d3d11_device_get_device_handle (self->device); + while (sample_count > 1) { + hr = device_handle->CheckMultisampleQualityLevels (desc.Format, + sample_count, &quality_levels); + if (gst_d3d11_result (hr, device) && quality_levels > 0) + break; + + sample_count = sample_count / 2; + }; + + if (sample_count > 1 && quality_levels > 0) { + ComPtr < ID3D11Texture2D > multisample_texture; + desc.SampleDesc.Count = sample_count; + desc.SampleDesc.Quality = quality_levels - 1; + device_handle->CreateTexture2D (&desc, nullptr, &multisample_texture); + + if (multisample_texture) { + msaa_mem = gst_d3d11_allocator_alloc_wrapped (nullptr, + self->device, multisample_texture.Get (), size, nullptr, nullptr); + + dmem = GST_D3D11_MEMORY_CAST (msaa_mem); + rtv = gst_d3d11_memory_get_render_target_view (dmem, 0); + } + } + context = gst_d3d11_device_get_device_context_handle (self->device); context->ClearRenderTargetView (rtv, clear_color); self->backbuffer = gst_buffer_new (); gst_buffer_append_memory (self->backbuffer, mem); + if (msaa_mem) { + self->msaa_buffer = gst_buffer_new (); + gst_buffer_append_memory (self->msaa_buffer, msaa_mem); + } + self->surface_width = desc.Width; self->surface_height = desc.Height; @@ -399,8 +462,10 @@ gst_d3d11_window_on_resize_default (GstD3D11Window * self, guint width, self->first_present = TRUE; /* redraw the last scene if cached buffer exits */ - if (self->cached_buffer) - gst_d3d11_window_present (self, self->cached_buffer, self->backbuffer); + if (self->cached_buffer) { + gst_d3d11_window_present (self, self->cached_buffer, self->backbuffer, + self->msaa_buffer); + } } void @@ -847,9 +912,123 @@ gst_d3d11_window_set_title (GstD3D11Window * window, const gchar * title) klass->set_title (window, title); } +static const XMFLOAT4X4 g_matrix_90r = XMFLOAT4X4 (0.0f, -1.0f, 0.0f, 0.0f, + 1.0f, 0.0f, 0.0f, 0.0f, + 0.0f, 0.0f, 1.0f, 0.0f, + 0.0f, 0.0f, 0.0f, 1.0f); + +static const XMFLOAT4X4 g_matrix_180 = XMFLOAT4X4 (-1.0f, 0.0f, 0.0f, 0.0f, + 0.0f, -1.0f, 0.0f, 0.0f, + 0.0f, 0.0f, 1.0f, 0.0f, + 0.0f, 0.0f, 0.0f, 1.0f); + +static const XMFLOAT4X4 g_matrix_90l = XMFLOAT4X4 (0.0f, 1.0f, 0.0f, 0.0f, + -1.0f, 0.0f, 0.0f, 0.0f, + 0.0f, 0.0f, 1.0f, 0.0f, + 0.0f, 0.0f, 0.0f, 1.0f); + +static const XMFLOAT4X4 g_matrix_horiz = XMFLOAT4X4 (-1.0f, 0.0f, 0.0f, 0.0f, + 0.0f, 1.0f, 0.0f, 0.0f, + 0.0f, 0.0f, 1.0f, 0.0f, + 0.0f, 0.0f, 0.0f, 1.0f); + +static const XMFLOAT4X4 g_matrix_vert = XMFLOAT4X4 (1.0f, 0.0f, 0.0f, 0.0f, + 0.0f, -1.0f, 0.0f, 0.0f, + 0.0f, 0.0f, 1.0f, 0.0f, + 0.0f, 0.0f, 0.0f, 1.0f); + +static const XMFLOAT4X4 g_matrix_ul_lr = XMFLOAT4X4 (0.0f, -1.0f, 0.0f, 0.0f, + -1.0f, 0.0f, 0.0f, 0.0f, + 0.0f, 0.0f, 1.0f, 0.0f, + 0.0f, 0.0f, 0.0f, 1.0f); + +static const XMFLOAT4X4 g_matrix_ur_ll = XMFLOAT4X4 (0.0f, 1.0f, 0.0f, 0.0f, + 1.0f, 0.0f, 0.0f, 0.0f, + 0.0f, 0.0f, 1.0f, 0.0f, + 0.0f, 0.0f, 0.0f, 1.0f); + +static void +gst_d3d11_window_calculate_matrix (GstD3D11Window * self, + gfloat viewport_width, gfloat viewport_height, gfloat transform_matrix[16]) +{ + gfloat aspect_ratio; + gboolean rotated = FALSE; + XMMATRIX rotate_matrix = XMMatrixIdentity (); + + switch (self->method) { + case GST_VIDEO_ORIENTATION_IDENTITY: + case GST_VIDEO_ORIENTATION_AUTO: + case GST_VIDEO_ORIENTATION_CUSTOM: + default: + break; + case GST_VIDEO_ORIENTATION_90R: + rotate_matrix = XMLoadFloat4x4 (&g_matrix_90r); + rotated = TRUE; + break; + case GST_VIDEO_ORIENTATION_180: + rotate_matrix = XMLoadFloat4x4 (&g_matrix_180); + break; + case GST_VIDEO_ORIENTATION_90L: + rotate_matrix = XMLoadFloat4x4 (&g_matrix_90l); + rotated = TRUE; + break; + case GST_VIDEO_ORIENTATION_HORIZ: + rotate_matrix = XMLoadFloat4x4 (&g_matrix_horiz); + break; + case GST_VIDEO_ORIENTATION_VERT: + rotate_matrix = XMLoadFloat4x4 (&g_matrix_vert); + break; + case GST_VIDEO_ORIENTATION_UL_LR: + rotate_matrix = XMLoadFloat4x4 (&g_matrix_ul_lr); + rotated = TRUE; + break; + case GST_VIDEO_ORIENTATION_UR_LL: + rotate_matrix = XMLoadFloat4x4 (&g_matrix_ur_ll); + rotated = TRUE; + break; + } + + if (rotated) + aspect_ratio = viewport_height / viewport_width; + else + aspect_ratio = viewport_width / viewport_height; + + /* Apply user specified transform matrix first, then rotate-method */ + XMMATRIX scale = + XMMatrixScaling (self->scale_x * aspect_ratio, self->scale_y, 1.0); + + XMMATRIX rotate = + XMMatrixRotationX (XMConvertToRadians (self->rotation_x)) * + XMMatrixRotationY (XMConvertToRadians (-self->rotation_y)) * + XMMatrixRotationZ (XMConvertToRadians (-self->rotation_z)); + + XMMATRIX view = XMMatrixLookAtLH (XMVectorSet (0.0, 0.0, -1.0, 0.0), + XMVectorSet (0.0, 0.0, 0.0, 0.0), XMVectorSet (0.0, 1.0, 0.0, 0.0)); + + XMMATRIX proj; + if (self->ortho) { + proj = XMMatrixOrthographicOffCenterLH (-aspect_ratio, + aspect_ratio, -1.0, 1.0, 0.1, 100.0); + } else { + proj = XMMatrixPerspectiveFovLH (XMConvertToRadians (self->fov), + aspect_ratio, 0.1, 100.0); + } + + XMMATRIX mvp = scale * rotate * view * proj * rotate_matrix; + + XMFLOAT4X4 matrix; + XMStoreFloat4x4 (&matrix, mvp); + + for (guint i = 0; i < 4; i++) { + for (guint j = 0; j < 4; j++) { + transform_matrix[i * 4 + j] = matrix.m[i][j]; + } + } +} + static GstFlowReturn gst_d3d11_window_present (GstD3D11Window * self, GstBuffer * buffer, - GstBuffer * backbuffer) + GstBuffer * backbuffer, GstBuffer * multisample) { GstD3D11WindowClass *klass = GST_D3D11_WINDOW_GET_CLASS (self); GstFlowReturn ret = GST_FLOW_OK; @@ -859,6 +1038,8 @@ gst_d3d11_window_present (GstD3D11Window * self, GstBuffer * buffer, ID3D11RenderTargetView *rtv; GstMemory *mem; GstD3D11Memory *dmem; + GstBuffer *target_buf; + ID3D11DeviceContext *context; if (!buffer) return GST_FLOW_OK; @@ -868,7 +1049,12 @@ gst_d3d11_window_present (GstD3D11Window * self, GstBuffer * buffer, return GST_FLOW_ERROR; } - mem = gst_buffer_peek_memory (backbuffer, 0); + if (multisample) + target_buf = multisample; + else + target_buf = backbuffer; + + mem = gst_buffer_peek_memory (target_buf, 0); if (!gst_is_d3d11_memory (mem)) { GST_ERROR_OBJECT (self, "Invalid back buffer"); return GST_FLOW_ERROR; @@ -881,14 +1067,13 @@ gst_d3d11_window_present (GstD3D11Window * self, GstBuffer * buffer, return GST_FLOW_ERROR; } + context = gst_d3d11_device_get_device_context_handle (self->device); + /* We use flip mode swapchain and will not redraw borders. * So backbuffer should be cleared manually in order to remove artifact of * previous client's rendering on present signal */ if (self->emit_present) { const FLOAT clear_color[] = { 0.0f, 0.0f, 0.0f, 1.0f }; - ID3D11DeviceContext *context = - gst_d3d11_device_get_device_context_handle (self->device); - context->ClearRenderTargetView (rtv, clear_color); } @@ -913,6 +1098,7 @@ gst_d3d11_window_present (GstD3D11Window * self, GstBuffer * buffer, if (self->first_present) { D3D11_VIEWPORT viewport; + const gfloat min_diff = 0.00001f; viewport.TopLeftX = self->render_rect.left; viewport.TopLeftY = self->render_rect.top; @@ -926,17 +1112,51 @@ gst_d3d11_window_present (GstD3D11Window * self, GstBuffer * buffer, "dest-width", (gint) (self->render_rect.right - self->render_rect.left), "dest-height", - (gint) (self->render_rect.bottom - self->render_rect.top), - "video-direction", self->method, nullptr); + (gint) (self->render_rect.bottom - self->render_rect.top), nullptr); + + if (!XMScalarNearEqual (self->rotation_x, 0.0f, min_diff) && + !XMScalarNearEqual (self->rotation_y, 0.0f, min_diff) && + !XMScalarNearEqual (self->rotation_z, 0.0f, min_diff) && + !XMScalarNearEqual (self->scale_x, 1.0f, min_diff) && + !XMScalarNearEqual (self->scale_y, 1.0f, min_diff)) { + g_object_set (self->converter, "video-direction", self->method, nullptr); + } else { + gfloat transform_matrix[16]; + + GST_DEBUG_OBJECT (self, "Applying custom transform"); + + gst_d3d11_window_calculate_matrix (self, + viewport.Width, viewport.Height, transform_matrix); + g_object_set (self->converter, + "video-direction", GST_VIDEO_ORIENTATION_CUSTOM, nullptr); + gst_d3d11_converter_set_transform_matrix (self->converter, + transform_matrix); + } + gst_d3d11_overlay_compositor_update_viewport (self->compositor, &viewport); } if (!gst_d3d11_converter_convert_buffer_unlocked (self->converter, - buffer, backbuffer)) { + buffer, target_buf)) { GST_ERROR_OBJECT (self, "Couldn't render buffer"); return GST_FLOW_ERROR; } + if (multisample) { + GstD3D11Memory *src_mem; + GstD3D11Memory *dst_mem; + + src_mem = (GstD3D11Memory *) gst_buffer_peek_memory (multisample, 0); + dst_mem = (GstD3D11Memory *) gst_buffer_peek_memory (backbuffer, 0); + + auto src_tex = gst_d3d11_memory_get_resource_handle (src_mem); + auto dst_tex = gst_d3d11_memory_get_resource_handle (dst_mem); + + context->ResolveSubresource (dst_tex, 0, src_tex, 0, self->dxgi_format); + + rtv = gst_d3d11_memory_get_render_target_view (dst_mem, 0); + } + gst_d3d11_overlay_compositor_upload (self->compositor, buffer); gst_d3d11_overlay_compositor_draw_unlocked (self->compositor, &rtv); @@ -963,7 +1183,7 @@ gst_d3d11_window_render (GstD3D11Window * window, GstBuffer * buffer) gst_buffer_replace (&window->cached_buffer, buffer); return gst_d3d11_window_present (window, window->cached_buffer, - window->backbuffer); + window->backbuffer, window->msaa_buffer); } GstFlowReturn @@ -993,7 +1213,7 @@ gst_d3d11_window_render_on_shared_handle (GstD3D11Window * window, return GST_FLOW_OK; } - ret = gst_d3d11_window_present (window, buffer, data.render_target); + ret = gst_d3d11_window_present (window, buffer, data.render_target, nullptr); klass->release_shared_handle (window, &data); @@ -1098,16 +1318,36 @@ gst_d3d11_window_get_native_type_to_string (GstD3D11WindowNativeType type) void gst_d3d11_window_set_orientation (GstD3D11Window * window, - GstVideoOrientationMethod method) + GstVideoOrientationMethod method, gfloat fov, gboolean ortho, + gfloat rotation_x, gfloat rotation_y, gfloat rotation_z, + gfloat scale_x, gfloat scale_y) { - if (method == GST_VIDEO_ORIENTATION_AUTO || - method == GST_VIDEO_ORIENTATION_CUSTOM) { - return; - } - GstD3D11DeviceLockGuard lk (window->device); - if (window->method != method) { + if (window->method != method || window->fov != fov || window->ortho != ortho + || window->rotation_x != rotation_x || window->rotation_y != rotation_y + || window->rotation_z != rotation_z || window->scale_x != scale_x + || window->scale_y != scale_y) { window->method = method; + window->fov = fov; + window->ortho = ortho; + window->rotation_x = rotation_x; + window->rotation_y = rotation_y; + window->rotation_z = rotation_z; + window->scale_x = scale_y; + if (window->swap_chain) { + GstD3D11WindowClass *klass = GST_D3D11_WINDOW_GET_CLASS (window); + + klass->on_resize (window, window->surface_width, window->surface_height); + } + } +} + +void +gst_d3d11_window_set_msaa_mode (GstD3D11Window * window, GstD3D11MSAAMode mode) +{ + GstD3D11DeviceLockGuard lk (window->device); + if (window->msaa != mode) { + window->msaa = mode; if (window->swap_chain) { GstD3D11WindowClass *klass = GST_D3D11_WINDOW_GET_CLASS (window); diff --git a/subprojects/gst-plugins-bad/sys/d3d11/gstd3d11window.h b/subprojects/gst-plugins-bad/sys/d3d11/gstd3d11window.h index 74dde52cce..1411057483 100644 --- a/subprojects/gst-plugins-bad/sys/d3d11/gstd3d11window.h +++ b/subprojects/gst-plugins-bad/sys/d3d11/gstd3d11window.h @@ -109,11 +109,20 @@ struct _GstD3D11Window IDXGISwapChain *swap_chain; GstBuffer *backbuffer; DXGI_FORMAT dxgi_format; + GstBuffer *msaa_buffer; GstBuffer *cached_buffer; gboolean first_present; GstVideoOrientationMethod method; + gfloat fov; + gboolean ortho; + gfloat rotation_x; + gfloat rotation_y; + gfloat rotation_z; + gfloat scale_x; + gfloat scale_y; + GstD3D11MSAAMode msaa; }; struct _GstD3D11WindowClass @@ -178,7 +187,17 @@ void gst_d3d11_window_set_title (GstD3D11Window * window, const gchar *title); void gst_d3d11_window_set_orientation (GstD3D11Window * window, - GstVideoOrientationMethod method); + GstVideoOrientationMethod method, + gfloat fov, + gboolean ortho, + gfloat rotation_x, + gfloat rotation_y, + gfloat rotation_z, + gfloat scale_x, + gfloat scale_y); + +void gst_d3d11_window_set_msaa_mode (GstD3D11Window * window, + GstD3D11MSAAMode mode); GstFlowReturn gst_d3d11_window_prepare (GstD3D11Window * window, guint display_width, diff --git a/subprojects/gst-plugins-bad/sys/d3d11/meson.build b/subprojects/gst-plugins-bad/sys/d3d11/meson.build index 40da8bb773..bdb1769e4f 100644 --- a/subprojects/gst-plugins-bad/sys/d3d11/meson.build +++ b/subprojects/gst-plugins-bad/sys/d3d11/meson.build @@ -130,6 +130,31 @@ if cc.get_id() == 'msvc' and fxc.found() extra_args += ['-DHLSL_PRECOMPILED'] endif +have_dx_math = cxx.compiles(''' + #include + #include + using namespace DirectX; + int main(int argc, char ** argv) { + XMMATRIX matrix; + XMFLOAT4X4 dump; + matrix = XMMatrixIdentity (); + XMStoreFloat4x4 (&dump, matrix); + return 0; + } + ''', + name: 'DirectXMath suupport in Windows SDK') + +if not have_dx_math + directxmath_dep = dependency('directxmath', + fallback: ['directxmath', 'directxmath_dep'], + required: d3d11_option) + if not directxmath_dep.found() + subdir_done () + endif + + extra_dep += [directxmath_dep] +endif + gstd3d11 = library('gstd3d11', d3d11_sources + hlsl_precompiled, c_args : gst_plugins_bad_args + extra_c_args + extra_args,