FaSTTUBe/GST-Tensordecoder-ov_ep
37
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

d3d12converter: Add support UV remap

Browse Source 
Adding OpenCV's cv::remap() like feature

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/8887>
This commit is contained in:
Seungha Yang 2025-04-20 23:33:16 +09:00
parent f600959f3b
commit 9b295cf7a2
5 changed files with 119 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
subprojects/gst-plugins-bad/gst-libs/gst/d3d11/gstd3d11converter.cpp
View File

@ -155,7 +155,7 @@ struct PSAlphaFactor
{
FLOAT alpha;
FLOAT padding[3];
FLOAT padding_other[4];
FLOAT padding_other[8];
};
struct PSConstBuffer
@ -2132,6 +2132,7 @@ gst_d3d11_converter_new (GstD3D11Device * device, const GstVideoInfo * in_info,
}
self->device = (GstD3D11Device *) gst_object_ref (device);
memset (&priv->alpha_data, 0, sizeof (priv->alpha_data));
priv->alpha_data.alpha = 1.0;
priv->in_info = *in_info;
priv->preproc_info = *in_info;

15
subprojects/gst-plugins-bad/gst-libs/gst/d3d12/gstd3d12converter-builder.cpp
View File

@ -181,6 +181,11 @@ ConverterRootSignature::ConverterRootSignature (D3D_ROOT_SIGNATURE_VERSION
D3D12_DESCRIPTOR_RANGE_FLAG_DATA_STATIC_WHILE_SET_AT_EXECUTE));
}
range_v1_1.push_back (CD3DX12_DESCRIPTOR_RANGE1
(D3D12_DESCRIPTOR_RANGE_TYPE_SRV, 1, 6, 0,
D3D12_DESCRIPTOR_RANGE_FLAG_DESCRIPTORS_VOLATILE |
D3D12_DESCRIPTOR_RANGE_FLAG_DATA_VOLATILE));
param.InitAsDescriptorTable (range_v1_1.size (),
range_v1_1.data (), D3D12_SHADER_VISIBILITY_PIXEL);
param_list_v1_1.push_back (param);
@ -190,11 +195,9 @@ ConverterRootSignature::ConverterRootSignature (D3D_ROOT_SIGNATURE_VERSION
sampler_range_v1_1.push_back (CD3DX12_DESCRIPTOR_RANGE1
(D3D12_DESCRIPTOR_RANGE_TYPE_SAMPLER, 1, 0, 0,
D3D12_DESCRIPTOR_RANGE_FLAG_DESCRIPTORS_VOLATILE));
if (build_lut) {
sampler_range_v1_1.push_back (CD3DX12_DESCRIPTOR_RANGE1
(D3D12_DESCRIPTOR_RANGE_TYPE_SAMPLER, 1, 1, 0,
D3D12_DESCRIPTOR_RANGE_FLAG_DESCRIPTORS_VOLATILE));
}
sampler_range_v1_1.push_back (CD3DX12_DESCRIPTOR_RANGE1
(D3D12_DESCRIPTOR_RANGE_TYPE_SAMPLER, 1, 1, 0,
D3D12_DESCRIPTOR_RANGE_FLAG_DESCRIPTORS_VOLATILE));
param.InitAsDescriptorTable (sampler_range_v1_1.size (),
sampler_range_v1_1.data (), D3D12_SHADER_VISIBILITY_PIXEL);
param_list_v1_1.push_back (param);
@ -206,7 +209,7 @@ ConverterRootSignature::ConverterRootSignature (D3D_ROOT_SIGNATURE_VERSION
/* PS alpha constant value, maybe updated */
ps_root_const_ = (UINT) param_list_v1_1.size ();
param.InitAsConstants (8, 1, 0, D3D12_SHADER_VISIBILITY_PIXEL);
param.InitAsConstants (12, 1, 0, D3D12_SHADER_VISIBILITY_PIXEL);
param_list_v1_1.push_back (param);
/* PS CBV, this is static */

4
subprojects/gst-plugins-bad/gst-libs/gst/d3d12/gstd3d12converter-private.h
View File

@ -44,4 +44,8 @@ gboolean gst_d3d12_converter_is_color_balance_needed (gfloat hue,
gfloat brightness,
gfloat contrast);
GST_D3D12_API
gboolean gst_d3d12_converter_set_remap (GstD3D12Converter * converter,
ID3D12Resource * remap_vector);
G_END_DECLS

74
subprojects/gst-plugins-bad/gst-libs/gst/d3d12/gstd3d12converter.cpp
View File

@ -167,8 +167,10 @@ struct PSConstBuffer
struct PSConstBufferDyn
{
float alphaFactor;
float padding[3];
UINT samplerRemap;
float padding[2];
float hsvcFactor[4];
float bgColor[4];
};
struct VertexData
@ -284,6 +286,7 @@ struct ConvertCtxCommon
ConvertCtxCommon()
{
const_data_dyn.alphaFactor = 1.0;
const_data_dyn.samplerRemap = 0;
const_data_dyn.hsvcFactor[0] = DEFAULT_HUE;
const_data_dyn.hsvcFactor[1] = DEFAULT_SATURATION;
const_data_dyn.hsvcFactor[2] = DEFAULT_BRIGHTNESS;
@ -317,6 +320,7 @@ struct ConvertCtxCommon
ComPtr<ID3D12Resource> gamma_enc_lut;
ComPtr<ID3D12DescriptorHeap> gamma_lut_heap;
ComPtr<ID3D12DescriptorHeap> sampler_heap;
ComPtr<ID3D12Resource> sampler_remap;
D3D12_VIEWPORT viewport[GST_VIDEO_MAX_PLANES];
D3D12_RECT scissor_rect[GST_VIDEO_MAX_PLANES];
ComPtr<ID3D12Fence> setup_fence;
@ -427,6 +431,7 @@ struct _GstD3D12ConverterPrivate
gboolean clear_background = FALSE;
FLOAT clear_color[4][4];
GstD3D12ColorMatrix clear_color_matrix;
GstD3D12ColorMatrix in_clear_color_matrix;
GstVideoOrientationMethod video_direction;
gboolean color_balance_enabled = FALSE;
@ -998,6 +1003,9 @@ gst_d3d12_converter_setup_resource (GstD3D12Converter * self,
if (have_lut)
num_srv += 2;
/* for sampler remap SRV */
num_srv++;
if (priv->max_srv_desc < num_srv)
priv->max_srv_desc = num_srv;
@ -1723,11 +1731,13 @@ gst_d3d12_converter_calculate_border_color (GstD3D12Converter * self)
{
auto priv = self->priv;
GstD3D12ColorMatrix *m = &priv->clear_color_matrix;
GstD3D12ColorMatrix *in_m = &priv->in_clear_color_matrix;
const GstVideoInfo *out_info = &priv->out_info;
gdouble a;
gdouble rgb[3];
gdouble converted[3];
GstVideoFormat format = GST_VIDEO_INFO_FORMAT (out_info);
auto comm = priv->main_ctx->comm;
a = ((priv->border_color & 0xffff000000000000) >> 48) / (gdouble) G_MAXUINT16;
rgb[0] =
@ -1738,11 +1748,17 @@ gst_d3d12_converter_calculate_border_color (GstD3D12Converter * self)
for (guint i = 0; i < 3; i++) {
converted[i] = 0;
comm->const_data_dyn.bgColor[i] = 0;
for (guint j = 0; j < 3; j++) {
converted[i] += m->matrix[i][j] * rgb[j];
comm->const_data_dyn.bgColor[i] += in_m->matrix[i][j] * rgb[j];
}
converted[i] += m->offset[i];
comm->const_data_dyn.bgColor[i] += in_m->offset[i];
converted[i] = CLAMP (converted[i], m->min[i], m->max[i]);
comm->const_data_dyn.bgColor[i] = CLAMP (comm->const_data_dyn.bgColor[i],
in_m->min[i], in_m->max[i]);
}
GST_DEBUG_OBJECT (self, "Calculated background color ARGB: %f, %f, %f, %f",
@ -2184,7 +2200,28 @@ gst_d3d12_converter_new (GstD3D12Device * device, GstD3D12CmdQueue * queue,
&yuv_info, &priv->clear_color_matrix);
}
gst_d3d12_converter_calculate_border_color (self);
if (GST_VIDEO_INFO_IS_RGB (&priv->in_info)) {
GstVideoInfo rgb_info = priv->in_info;
rgb_info.colorimetry.range = GST_VIDEO_COLOR_RANGE_0_255;
gst_d3d12_color_range_adjust_matrix_unorm (&rgb_info, &priv->in_info,
&priv->in_clear_color_matrix);
} else {
GstVideoInfo rgb_info;
GstVideoInfo yuv_info;
gst_video_info_set_format (&rgb_info, GST_VIDEO_FORMAT_RGBA64_LE,
priv->in_info.width, priv->in_info.height);
convert_info_gray_to_yuv (&priv->in_info, &yuv_info);
if (yuv_info.colorimetry.matrix == GST_VIDEO_COLOR_MATRIX_UNKNOWN ||
yuv_info.colorimetry.matrix == GST_VIDEO_COLOR_MATRIX_RGB) {
GST_WARNING_OBJECT (self, "Invalid matrix is detected");
yuv_info.colorimetry.matrix = GST_VIDEO_COLOR_MATRIX_BT709;
}
gst_d3d12_rgb_to_yuv_matrix_unorm (&rgb_info,
&yuv_info, &priv->in_clear_color_matrix);
}
PSConstBuffer const_data[2];
CONVERT_TYPE convert_type[2];
@ -2325,6 +2362,8 @@ gst_d3d12_converter_new (GstD3D12Device * device, GstD3D12CmdQueue * queue,
}
}
gst_d3d12_converter_calculate_border_color (self);
D3D12_DESCRIPTOR_HEAP_DESC srv_heap_desc = { };
srv_heap_desc.Type = D3D12_DESCRIPTOR_HEAP_TYPE_CBV_SRV_UAV;
srv_heap_desc.NumDescriptors = priv->max_srv_desc;
@ -2509,6 +2548,12 @@ gst_d3d12_converter_execute (GstD3D12Converter * self, GstD3D12Frame * in_frame,
device->CopyDescriptorsSimple (2, cpu_handle,
GetCPUDescriptorHandleForHeapStart (comm->gamma_lut_heap),
D3D12_DESCRIPTOR_HEAP_TYPE_CBV_SRV_UAV);
cpu_handle.Offset (priv->srv_inc_size);
}
if (comm->const_data_dyn.samplerRemap) {
device->CreateShaderResourceView (comm->sampler_remap.Get (),
nullptr, cpu_handle);
}
if (priv->clear_background) {
@ -2578,6 +2623,12 @@ gst_d3d12_converter_execute (GstD3D12Converter * self, GstD3D12Frame * in_frame,
FENCE_NOTIFY_COM (ctx->vertex_upload.Detach ()));
}
if (ctx->comm->sampler_remap) {
ComPtr < ID3D12Resource > remap_clone = ctx->comm->sampler_remap;
gst_d3d12_fence_data_push (fence_data,
FENCE_NOTIFY_COM (remap_clone.Detach ()));
}
auto sampler = comm->sampler_heap.Get ();
sampler->AddRef ();
gst_d3d12_fence_data_push (fence_data, FENCE_NOTIFY_COM (sampler));
@ -3053,3 +3104,22 @@ gst_d3d12_converter_is_color_balance_needed (gfloat hue, gfloat saturation,
return FALSE;
}
gboolean
gst_d3d12_converter_set_remap (GstD3D12Converter * converter,
ID3D12Resource * remap_vector)
{
g_return_val_if_fail (GST_IS_D3D12_CONVERTER (converter), FALSE);
auto priv = converter->priv;
std::lock_guard < std::mutex > lk (priv->prop_lock);
auto comm = priv->main_ctx->comm;
comm->sampler_remap = remap_vector;
if (remap_vector)
comm->const_data_dyn.samplerRemap = 1;
else
comm->const_data_dyn.samplerRemap = 0;
return TRUE;
}

36
subprojects/gst-plugins-bad/gst-libs/gst/d3dshader/converter-hlsl/PSMain_converter.hlsl
View File

@ -21,8 +21,10 @@
cbuffer PsConstBufferDyn : register(b1)
{
float alphaFactor;
float3 padding_0;
uint remapUV;
float2 padding_0;
float4 hsvcFactor;
float4 bg_color;
};
struct PSColorSpace
@ -49,6 +51,7 @@ Texture2D shaderTexture_2 : register(t2);
Texture2D shaderTexture_3 : register(t3);
Texture1D<float> gammaDecLUT : register(t4);
Texture1D<float> gammaEncLUT: register(t5);
Texture2D samplerRemap : register(t6);
SamplerState samplerState : register(s0);
SamplerState lutSamplerState : register(s1);
@ -1460,15 +1463,28 @@ OUTPUT_TYPE ENTRY_POINT (PS_INPUT input)
SAMPLER g_sampler;
CONVERTER g_converter;
OUTPUT_BUILDER g_builder;
return g_builder.Build (g_converter.Execute (g_sampler.Execute (input.Texture)));
float2 uv;
[branch] if (remapUV) {
float4 val = samplerRemap.Sample(lutSamplerState, input.Texture);
if (val.w < 0.5)
return g_builder.Build (g_converter.Execute (bg_color));
uv = val.xy;
} else {
uv = input.Texture;
}
return g_builder.Build (g_converter.Execute (g_sampler.Execute (uv)));
}
#else /* BUILDING_HLSL */
static const char str_PSMain_converter[] =
"cbuffer PsConstBufferDyn : register(b1)\n"
"{\n"
" float alphaFactor;\n"
" float3 padding_0;\n"
" uint remapUV;\n"
" float2 padding_0;\n"
" float4 hsvcFactor;\n"
" float4 bg_color;\n"
"};\n"
"\n"
"struct PSColorSpace\n"
@ -1495,6 +1511,7 @@ static const char str_PSMain_converter[] =
"Texture2D shaderTexture_3 : register(t3);\n"
"Texture1D<float> gammaDecLUT : register(t4);\n"
"Texture1D<float> gammaEncLUT: register(t5);\n"
"Texture2D samplerRemap : register(t6);\n"
"\n"
"SamplerState samplerState : register(s0);\n"
"SamplerState lutSamplerState : register(s1);\n"
@ -2906,6 +2923,17 @@ static const char str_PSMain_converter[] =
" SAMPLER g_sampler;\n"
" CONVERTER g_converter;\n"
" OUTPUT_BUILDER g_builder;\n"
" return g_builder.Build (g_converter.Execute (g_sampler.Execute (input.Texture)));\n"
" float2 uv;\n"
" [branch] if (remapUV) {\n"
" float4 val = samplerRemap.Sample(lutSamplerState, input.Texture);\n"
" if (val.w < 0.5)\n"
" return g_builder.Build (g_converter.Execute (bg_color));\n"
"\n"
" uv = val.xy;\n"
" } else {\n"
" uv = input.Texture;\n"
" }\n"
"\n"
" return g_builder.Build (g_converter.Execute (g_sampler.Execute (uv)));\n"
"}\n";
#endif