FaSTTUBe/GST-Tensordecoder-ov_ep
37
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

d3d12: Add support for dewarping fisheye images

Browse Source 
Add d3d12fisheyedewarp element that performs fisheye image dewarping
using D3D12. A UV remap LUT texture is generated via a compute shader,
and the actual remapping is performed in a pixel shader using this LUT

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/9441>
This commit is contained in:
Seungha Yang 2025-06-20 03:23:25 +09:00
parent 316473f722
commit 461e3946d4
11 changed files with 1923 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
subprojects/gst-plugins-bad/gst-libs/gst/d3dshader/gstd3dshadercache.cpp
View File

@ -96,6 +96,9 @@ static const ShaderItem g_cs_map[] = {
{GST_D3D_PLUGIN_CS_YADIF_1_12, BUILD_SOURCE (CSMain_yadif_1_12)},
{GST_D3D_PLUGIN_CS_YADIF_2, BUILD_SOURCE (CSMain_yadif_2)},
{GST_D3D_PLUGIN_CS_YADIF_4, BUILD_SOURCE (CSMain_yadif_4)},
{GST_D3D_PLUGIN_CS_FISHEYE_EQUIRECT, BUILD_SOURCE (CSMain_fisheye_equirect)},
{GST_D3D_PLUGIN_CS_FISHEYE_PANORAMA, BUILD_SOURCE (CSMain_fisheye_panorama)},
{GST_D3D_PLUGIN_CS_FISHEYE_PERSPECTIVE, BUILD_SOURCE (CSMain_fisheye_perspective)},
};
#undef BUILD_SOURCE

3
subprojects/gst-plugins-bad/gst-libs/gst/d3dshader/gstd3dshadercache.h
View File

@ -62,6 +62,9 @@ typedef enum
GST_D3D_PLUGIN_CS_YADIF_1_12,
GST_D3D_PLUGIN_CS_YADIF_2,
GST_D3D_PLUGIN_CS_YADIF_4,
GST_D3D_PLUGIN_CS_FISHEYE_EQUIRECT,
GST_D3D_PLUGIN_CS_FISHEYE_PANORAMA,
GST_D3D_PLUGIN_CS_FISHEYE_PERSPECTIVE,
GST_D3D_PLUGIN_CS_LAST,
} GstD3DPluginCS;

167
subprojects/gst-plugins-bad/gst-libs/gst/d3dshader/plugin-hlsl/CSMain_fisheye_equirect.hlsl Normal file
View File

@ -0,0 +1,167 @@
/* GStreamer
* Copyright (C) 2025 Seungha Yang <seungha@centricular.com>
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the
* Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
* Boston, MA 02110-1301, USA.
*/
#ifdef BUILDING_HLSL
RWTexture2D<float4> uvLUT : register(u0);
cbuffer Parameters : register(b0)
{
float2 fisheyeCenter;
float2 fisheyeRadius;
float maxAngle;
float horizontalFOV;
float verticalFOV;
float rollAngle; // unused
float2 roiOffset;
float2 roiScale;
float4 padding;
float3x3 RotationMatrix;
};
[numthreads(8, 8, 1)]
void ENTRY_POINT (uint3 DTid : SV_DispatchThreadID)
{
uint width, height;
uvLUT.GetDimensions(width, height);
if (DTid.x >= width || DTid.y >= height)
return;
// Compute normalized screen coordinate
float2 uv = float2(DTid.x, DTid.y) / float2(width, height);
// Apply ROI cropping and scaling
float2 uv_roi = roiOffset + uv * roiScale;
// Convert to spherical coordinates (delta = latitude, psi = longitude)
float delta = verticalFOV * (uv_roi.y - 0.5); // up-down angle
float psi = horizontalFOV * (uv_roi.x - 0.5); // left-right angle
// Convert spherical to 3D ray (Z-forward)
float cosD = cos(delta);
float sinD = sin(delta);
float cosP = cos(psi);
float sinP = sin(psi);
float3 ray = float3(
cosD * sinP, // X
sinD, // Y
cosD * cosP // Z
);
// Apply rotation matrix
float3 rotatedRay = mul(RotationMatrix, ray);
rotatedRay = normalize(rotatedRay);
// Convert back to spherical angles
float theta = acos(rotatedRay.z); // zenith angle
float4 fishUV = float4(0.0, 0.0, 0.0, 1.0);
if (theta <= maxAngle) {
// azimuth angle
float phi = atan2(rotatedRay.y, rotatedRay.x);
// Map to fisheye UV via equidistant projection
float2 r = (fisheyeRadius / maxAngle) * theta;
fishUV.xy = fisheyeCenter + r * float2(cos(phi), sin(phi));
} else {
// Out of view
fishUV.w = 0.0;
}
uvLUT[DTid.xy] = fishUV;
}
#else
static const char str_CSMain_fisheye_equirect[] =
"RWTexture2D<float4> uvLUT : register(u0);\n"
"\n"
"cbuffer Parameters : register(b0)\n"
"{\n"
" float2 fisheyeCenter;\n"
" float2 fisheyeRadius;\n"
"\n"
" float maxAngle;\n"
" float horizontalFOV;\n"
" float verticalFOV;\n"
" float rollAngle; // unused\n"
"\n"
" float2 roiOffset;\n"
" float2 roiScale;\n"
"\n"
" float4 padding;\n"
"\n"
" float3x3 RotationMatrix;\n"
"};\n"
"\n"
"[numthreads(8, 8, 1)]\n"
"void ENTRY_POINT (uint3 DTid : SV_DispatchThreadID)\n"
"{\n"
" uint width, height;\n"
" uvLUT.GetDimensions(width, height);\n"
" if (DTid.x >= width || DTid.y >= height)\n"
" return;\n"
"\n"
" // Compute normalized screen coordinate\n"
" float2 uv = float2(DTid.x, DTid.y) / float2(width, height);\n"
"\n"
" // Apply ROI cropping and scaling\n"
" float2 uv_roi = roiOffset + uv * roiScale;\n"
"\n"
" // Convert to spherical coordinates (delta = latitude, psi = longitude)\n"
" float delta = verticalFOV * (uv_roi.y - 0.5); // up-down angle\n"
" float psi = horizontalFOV * (uv_roi.x - 0.5); // left-right angle\n"
"\n"
" // Convert spherical to 3D ray (Z-forward)\n"
" float cosD = cos(delta);\n"
" float sinD = sin(delta);\n"
" float cosP = cos(psi);\n"
" float sinP = sin(psi);\n"
"\n"
" float3 ray = float3(\n"
" cosD * sinP, // X\n"
" sinD, // Y\n"
" cosD * cosP // Z\n"
" );\n"
"\n"
" // Apply rotation matrix\n"
" float3 rotatedRay = mul(RotationMatrix, ray);\n"
" rotatedRay = normalize(rotatedRay);\n"
"\n"
" // Convert back to spherical angles\n"
" float theta = acos(rotatedRay.z); // zenith angle\n"
"\n"
" float4 fishUV = float4(0.0, 0.0, 0.0, 1.0);\n"
" if (theta <= maxAngle) {\n"
" // azimuth angle\n"
" float phi = atan2(rotatedRay.y, rotatedRay.x);\n"
"\n"
" // Map to fisheye UV via equidistant projection\n"
" float2 r = (fisheyeRadius / maxAngle) * theta;\n"
" fishUV.xy = fisheyeCenter + r * float2(cos(phi), sin(phi));\n"
" } else {\n"
" // Out of view\n"
" fishUV.w = 0.0;\n"
" }\n"
"\n"
" uvLUT[DTid.xy] = fishUV;\n"
"}\n";
#endif

131
subprojects/gst-plugins-bad/gst-libs/gst/d3dshader/plugin-hlsl/CSMain_fisheye_panorama.hlsl Normal file
View File

@ -0,0 +1,131 @@
/* GStreamer
* Copyright (C) 2025 Seungha Yang <seungha@centricular.com>
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the
* Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
* Boston, MA 02110-1301, USA.
*/
#ifdef BUILDING_HLSL
RWTexture2D<float4> uvLUT : register(u0);
cbuffer Parameters : register(b0)
{
float2 fisheyeCenter;
float2 fisheyeRadius;
float maxAngle;
float horizontalFOV; // Unused
float verticalFOV; // Unused
float rollAngle;
float2 roiOffset;
float2 roiScale;
float innerRadius;
float3 padding;
float3x3 RotationMatrix; // Unused
};
[numthreads(8, 8, 1)]
void ENTRY_POINT (uint3 DTid : SV_DispatchThreadID)
{
uint width, height;
uvLUT.GetDimensions(width, height);
if (DTid.x >= width || DTid.y >= height)
return;
// Compute normalized screen coordinate
float2 uv = float2(DTid.xy) / float2(width, height);
// Apply ROI cropping and scaling
float2 uv_roi = roiOffset + uv * roiScale;
// Zenith angle (theta): 0 = center, maxAngle = outer edge
float minTheta = maxAngle * saturate(innerRadius);
float theta = lerp(minTheta, maxAngle, 1.0 - uv_roi.y);
// Map to azimuthal angle (phi) across full 360 degrees
float phi = -6.28318530718 * (uv_roi.x - 0.5) + rollAngle;
float4 fishUV = float4(0.0, 0.0, 0.0, 1.0);
if (theta >= minTheta && theta <= maxAngle) {
// Convert spherical coordinates to 2D fisheye UV using equidistant projection
float2 r = (fisheyeRadius / maxAngle) * theta;
fishUV.xy = fisheyeCenter + r * float2(cos(phi), -sin(phi));
} else {
// Out of view
fishUV.w = 0.0;
}
uvLUT[DTid.xy] = fishUV;
}
#else
static const char str_CSMain_fisheye_panorama[] =
"RWTexture2D<float4> uvLUT : register(u0);\n"
"\n"
"cbuffer Parameters : register(b0)\n"
"{\n"
" float2 fisheyeCenter;\n"
" float2 fisheyeRadius;\n"
"\n"
" float maxAngle;\n"
" float horizontalFOV; // Unused\n"
" float verticalFOV; // Unused\n"
" float rollAngle;\n"
"\n"
" float2 roiOffset;\n"
" float2 roiScale;\n"
"\n"
" float innerRadius;\n"
" float3 padding;\n"
"\n"
" float3x3 RotationMatrix; // Unused\n"
"};\n"
"\n"
"[numthreads(8, 8, 1)]\n"
"void ENTRY_POINT (uint3 DTid : SV_DispatchThreadID)\n"
"{\n"
" uint width, height;\n"
" uvLUT.GetDimensions(width, height);\n"
" if (DTid.x >= width || DTid.y >= height)\n"
" return;\n"
"\n"
" // Compute normalized screen coordinate\n"
" float2 uv = float2(DTid.xy) / float2(width, height);\n"
"\n"
" // Apply ROI cropping and scaling\n"
" float2 uv_roi = roiOffset + uv * roiScale;\n"
"\n"
" // Zenith angle (theta): 0 = center, maxAngle = outer edge\n"
" float minTheta = maxAngle * saturate(innerRadius);\n"
" float theta = lerp(minTheta, maxAngle, 1.0 - uv_roi.y);\n"
"\n"
" // Map to azimuthal angle (phi) across full 360 degrees\n"
" float phi = -6.28318530718 * (uv_roi.x - 0.5) + rollAngle;\n"
"\n"
" float4 fishUV = float4(0.0, 0.0, 0.0, 1.0);\n"
" if (theta >= minTheta && theta <= maxAngle) {\n"
" // Convert spherical coordinates to 2D fisheye UV using equidistant projection\n"
" float2 r = (fisheyeRadius / maxAngle) * theta;\n"
" fishUV.xy = fisheyeCenter + r * float2(cos(phi), -sin(phi));\n"
" } else {\n"
" // Out of view\n"
" fishUV.w = 0.0;\n"
" }\n"
"\n"
" uvLUT[DTid.xy] = fishUV;\n"
"}\n";
#endif

151
subprojects/gst-plugins-bad/gst-libs/gst/d3dshader/plugin-hlsl/CSMain_fisheye_perspective.hlsl Normal file
View File

@ -0,0 +1,151 @@
/* GStreamer
* Copyright (C) 2025 Seungha Yang <seungha@centricular.com>
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the
* Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
* Boston, MA 02110-1301, USA.
*/
#ifdef BUILDING_HLSL
RWTexture2D<float4> uvLUT : register(u0);
cbuffer Parameters : register(b0)
{
float2 fisheyeCenter;
float2 fisheyeRadius;
float maxAngle;
float horizontalFOV; // unused
float verticalFOV; // unused
float rollAngle; // Unused
float2 roiOffset;
float2 roiScale;
float padding;
float invFocalLenX;
float invFocalLenY;
float otherPadding;
float3x3 RotationMatrix;
};
[numthreads(8, 8, 1)]
void ENTRY_POINT (uint3 DTid : SV_DispatchThreadID)
{
uint width, height;
uvLUT.GetDimensions(width, height);
if (DTid.x >= width || DTid.y >= height)
return;
// Compute normalized screen coordinate
float2 uv = float2(DTid.xy) / float2(width, height);
// Apply ROI cropping and scaling
float2 uv_roi = roiOffset + uv * roiScale;
// Convert to NDC [-1, 1]
float2 uv_ndc = uv_roi * 2.0 - 1.0;
// Compute view ray from perspective FOV (pinhole model)
float x = invFocalLenX * uv_ndc.x;
float y = invFocalLenY * uv_ndc.y;
float3 localRay = normalize(float3(x, y, 1.0));
float3 worldRay = normalize(mul(RotationMatrix, localRay));
// Compute angle from Z-axis (zenith angle)
float angle = acos(worldRay.z);
float4 fishUV = float4(0.0, 0.0, 0.0, 1.0);
if (angle <= maxAngle) {
// Project to fisheye image using equidistant projection
float phi = atan2(worldRay.y, worldRay.x);
float2 r = (fisheyeRadius / maxAngle) * angle;
fishUV.xy = fisheyeCenter + r * float2(cos(phi), sin(phi));
} else {
// Out of view
fishUV.w = 0.0;
}
uvLUT[DTid.xy] = fishUV;
}
#else
static const char str_CSMain_fisheye_perspective[] =
"RWTexture2D<float4> uvLUT : register(u0);\n"
"\n"
"cbuffer Parameters : register(b0)\n"
"{\n"
" float2 fisheyeCenter;\n"
" float2 fisheyeRadius;\n"
"\n"
" float maxAngle;\n"
" float horizontalFOV; // unused\n"
" float verticalFOV; // unused\n"
" float rollAngle; // Unused\n"
"\n"
" float2 roiOffset;\n"
" float2 roiScale;\n"
"\n"
" float padding;\n"
" float invFocalLenX;\n"
" float invFocalLenY;\n"
" float otherPadding;\n"
"\n"
" float3x3 RotationMatrix;\n"
"};\n"
"\n"
"[numthreads(8, 8, 1)]\n"
"void ENTRY_POINT (uint3 DTid : SV_DispatchThreadID)\n"
"{\n"
" uint width, height;\n"
" uvLUT.GetDimensions(width, height);\n"
" if (DTid.x >= width || DTid.y >= height)\n"
" return;\n"
"\n"
" // Compute normalized screen coordinate\n"
" float2 uv = float2(DTid.xy) / float2(width, height);\n"
"\n"
" // Apply ROI cropping and scaling\n"
" float2 uv_roi = roiOffset + uv * roiScale;\n"
"\n"
" // Convert to NDC [-1, 1]\n"
" float2 uv_ndc = uv_roi * 2.0 - 1.0;\n"
"\n"
" // Compute view ray from perspective FOV (pinhole model)\n"
" float x = invFocalLenX * uv_ndc.x;\n"
" float y = invFocalLenY * uv_ndc.y;\n"
" float3 localRay = normalize(float3(x, y, 1.0));\n"
"\n"
" float3 worldRay = normalize(mul(RotationMatrix, localRay));\n"
"\n"
" // Compute angle from Z-axis (zenith angle)\n"
" float angle = acos(worldRay.z);\n"
"\n"
" float4 fishUV = float4(0.0, 0.0, 0.0, 1.0);\n"
" if (angle <= maxAngle) {\n"
" // Project to fisheye image using equidistant projection\n"
" float phi = atan2(worldRay.y, worldRay.x);\n"
"\n"
" float2 r = (fisheyeRadius / maxAngle) * angle;\n"
" fishUV.xy = fisheyeCenter + r * float2(cos(phi), sin(phi));\n"
" } else {\n"
" // Out of view\n"
" fishUV.w = 0.0;\n"
" }\n"
"\n"
" uvLUT[DTid.xy] = fishUV;\n"
"}\n";
#endif

3
subprojects/gst-plugins-bad/gst-libs/gst/d3dshader/plugin-hlsl/hlsl.h
View File

@ -41,3 +41,6 @@
#include "CSMain_yadif_1_12.hlsl"
#include "CSMain_yadif_2.hlsl"
#include "CSMain_yadif_4.hlsl"
#include "CSMain_fisheye_equirect.hlsl"
#include "CSMain_fisheye_panorama.hlsl"
#include "CSMain_fisheye_perspective.hlsl"

3
subprojects/gst-plugins-bad/gst-libs/gst/d3dshader/plugin-hlsl/meson.build
View File

@ -21,6 +21,9 @@ hlsl_sources = [
['CSMain_yadif_1', 'cs'],
['CSMain_yadif_2', 'cs'],
['CSMain_yadif_4', 'cs'],
['CSMain_fisheye_equirect', 'cs'],
['CSMain_fisheye_panorama', 'cs'],
['CSMain_fisheye_perspective', 'cs'],
]
shader_model = '5_0'

1425
subprojects/gst-plugins-bad/sys/d3d12/gstd3d12fisheyedewarp.cpp Normal file
View File

File diff suppressed because it is too large Load Diff

32
subprojects/gst-plugins-bad/sys/d3d12/gstd3d12fisheyedewarp.h Normal file
View File

@ -0,0 +1,32 @@
/* GStreamer
* Copyright (C) 2025 Seungha Yang <seungha@centricular.com>
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the
* Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
* Boston, MA 02110-1301, USA.
*/
#pragma once
#include <gst/gst.h>
#include "gstd3d12basefilter.h"
G_BEGIN_DECLS
#define GST_TYPE_D3D12_FISHEYE_DEWARP (gst_d3d12_fisheye_dewarp_get_type())
G_DECLARE_FINAL_TYPE (GstD3D12FisheyeDewarp, gst_d3d12_fisheye_dewarp,
GST, D3D12_FISHEYE_DEWARP, GstD3D12BaseFilter)
G_END_DECLS

2
subprojects/gst-plugins-bad/sys/d3d12/meson.build
View File

@ -35,6 +35,7 @@ d3d12_sources = [
'gstd3d12window-swapchain.cpp',
'gstd3d12window-win32.cpp',
'gstd3d12window.cpp',
'gstd3d12fisheyedewarp.cpp',
'plugin.cpp',
]
@ -81,6 +82,7 @@ d3d12_headers = [
'gstd3d12basefilter.h',
'gstd3d12testsrc.h',
'gstd3d12vp8dec.h',
'gstd3d12fisheyedewarp.h',
]
graphicscapture_sources = [

3
subprojects/gst-plugins-bad/sys/d3d12/plugin.cpp
View File

@ -52,6 +52,7 @@
#include "gstd3d12mipmapping.h"
#include "gstd3d12deinterlace.h"
#include "gstd3d12remap.h"
#include "gstd3d12fisheyedewarp.h"
#include <windows.h>
#include <versionhelpers.h>
#include <wrl.h>
@ -202,6 +203,8 @@ plugin_init (GstPlugin * plugin)
"d3d12deinterlace", GST_RANK_NONE, GST_TYPE_D3D12_DEINTERLACE);
gst_element_register (plugin,
"d3d12remap", GST_RANK_NONE, GST_TYPE_D3D12_REMAP);
gst_element_register (plugin,
"d3d12fisheyedewarp", GST_RANK_NONE, GST_TYPE_D3D12_FISHEYE_DEWARP);
g_object_set_data_full (G_OBJECT (plugin),
"plugin-d3d12-shutdown", (gpointer) "shutdown-data",