d3d12swapchainsink: Add last-rendered-sample action signal
Add a new action signal to allow applications to capture the most recently rendered frame directly from the swapchain back buffer. Unlike the existing "last-sample" property, which exposes the raw input sample before any sink-side processing, this signal captures the final displayed image after any internal image processing (e.g., UV remap, color balance, overlay) has been applied. Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/9423>
This commit is contained in:
Seungha Yang
GStreamer Marge Bot
parent
559dbdecf1
commit
4f932ee081
@ -86,6 +86,7 @@ enum
|
||||
SIGNAL_RESIZE,
|
||||
SIGNAL_UV_REMAP,
|
||||
SIGNAL_REDRAW,
|
||||
SIGNAL_LAST_RENDERED_SAMPLE,
|
||||
SIGNAL_LAST
|
||||
};
|
||||
|
||||
@ -229,6 +230,10 @@ struct GstD3D12SwapChainSinkPrivate
|
||||
GstVideoRectangle viewport = { };
|
||||
gboolean auto_resize = FALSE;
|
||||
gboolean did_redraw = FALSE;
|
||||
guint last_backbuf_idx = 0;
|
||||
GstClockTime last_backbuf_pts = 0;
|
||||
GstClockTime last_backbuf_dur = 0;
|
||||
GstSegment segment;
|
||||
|
||||
std::vector<ComPtr<ID3D12Resource>> uv_remap;
|
||||
std::vector<D3D12_VIEWPORT> uv_remap_viewport_origin;
|
||||
@ -275,6 +280,8 @@ static gboolean gst_d3d12_swapchain_sink_query (GstBaseSink * sink,
|
||||
GstQuery * query);
|
||||
static GstFlowReturn gst_d3d12_swapchain_sink_prepare (GstBaseSink * sink,
|
||||
GstBuffer * buf);
|
||||
static gboolean gst_d3d12_swapchain_sink_event (GstBaseSink * sink,
|
||||
GstEvent * event);
|
||||
static gboolean gst_d3d12_swapchain_sink_set_info (GstVideoSink * sink,
|
||||
GstCaps * caps, const GstVideoInfo * info);
|
||||
static GstFlowReturn gst_d3d12_swapchain_sink_show_frame (GstVideoSink * sink,
|
||||
@ -288,6 +295,9 @@ static void gst_d3d12_swapchain_sink_redraw (GstD3D12SwapChainSink * self);
|
||||
static void
|
||||
gst_d3d12_swapchain_sink_resize_internal (GstD3D12SwapChainSink * self,
|
||||
guint width, guint height);
|
||||
static GstSample
|
||||
* gst_d3d12_swapchain_sink_last_back_buffer (GstD3D12SwapChainSink * self,
|
||||
gboolean remove_borders);
|
||||
|
||||
static void
|
||||
gst_d3d12_swapchain_sink_color_balance_init (GstColorBalanceInterface * iface);
|
||||
@ -441,6 +451,25 @@ gst_d3d12_swapchain_sink_class_init (GstD3D12SwapChainSinkClass * klass)
|
||||
G_CALLBACK (gst_d3d12_swapchain_sink_redraw), nullptr, nullptr, nullptr,
|
||||
G_TYPE_NONE, 0);
|
||||
|
||||
/**
|
||||
* GstD3D12SwapChainSink::last-rendered-sample:
|
||||
* @videosink: the #GstD3D12SwapChainSink
|
||||
* @remove_borders: Remove background borders
|
||||
*
|
||||
* Get last rendered swapchain backbuffer content
|
||||
*
|
||||
* Returns: a #GstSample of the last rendered swapchain backbuffer content
|
||||
* or %NULL if swapchain is not configured yet
|
||||
*
|
||||
* Since: 1.28
|
||||
*/
|
||||
d3d12_swapchain_sink_signals[SIGNAL_LAST_RENDERED_SAMPLE] =
|
||||
g_signal_new_class_handler ("last-rendered-sample",
|
||||
G_TYPE_FROM_CLASS (klass),
|
||||
(GSignalFlags) (G_SIGNAL_RUN_LAST | G_SIGNAL_ACTION),
|
||||
G_CALLBACK (gst_d3d12_swapchain_sink_last_back_buffer),
|
||||
nullptr, nullptr, nullptr, GST_TYPE_SAMPLE, 1, G_TYPE_BOOLEAN);
|
||||
|
||||
element_class->set_context =
|
||||
GST_DEBUG_FUNCPTR (gst_d3d12_swapchain_sink_set_context);
|
||||
|
||||
@ -458,6 +487,7 @@ gst_d3d12_swapchain_sink_class_init (GstD3D12SwapChainSinkClass * klass)
|
||||
basesink_class->query = GST_DEBUG_FUNCPTR (gst_d3d12_swapchain_sink_query);
|
||||
basesink_class->prepare =
|
||||
GST_DEBUG_FUNCPTR (gst_d3d12_swapchain_sink_prepare);
|
||||
basesink_class->event = GST_DEBUG_FUNCPTR (gst_d3d12_swapchain_sink_event);
|
||||
|
||||
videosink_class->set_info =
|
||||
GST_DEBUG_FUNCPTR (gst_d3d12_swapchain_sink_set_info);
|
||||
@ -801,6 +831,20 @@ gst_d3d12_swapchain_sink_set_context (GstElement * element,
|
||||
GST_ELEMENT_CLASS (parent_class)->set_context (element, context);
|
||||
}
|
||||
|
||||
static gboolean
|
||||
gst_d3d12_swapchain_sink_event (GstBaseSink * sink, GstEvent * event)
|
||||
{
|
||||
auto self = GST_D3D12_SWAPCHAIN_SINK (sink);
|
||||
auto priv = self->priv;
|
||||
|
||||
if (GST_EVENT_TYPE (event) == GST_EVENT_SEGMENT) {
|
||||
std::lock_guard < std::recursive_mutex > lk (priv->lock);
|
||||
gst_event_copy_segment (event, &priv->segment);
|
||||
}
|
||||
|
||||
return GST_BASE_SINK_CLASS (parent_class)->event (sink, event);
|
||||
}
|
||||
|
||||
static gboolean
|
||||
gst_d3d12_swapchain_sink_set_info (GstVideoSink * sink, GstCaps * caps,
|
||||
const GstVideoInfo * info)
|
||||
@ -1053,8 +1097,10 @@ gst_d3d12_swapchain_sink_render (GstD3D12SwapChainSink * self)
|
||||
}
|
||||
}
|
||||
|
||||
auto cur_idx = priv->swapchain->GetCurrentBackBufferIndex ();
|
||||
auto backbuf = priv->backbuf[cur_idx]->backbuf;
|
||||
priv->last_backbuf_idx = priv->swapchain->GetCurrentBackBufferIndex ();
|
||||
priv->last_backbuf_pts = GST_BUFFER_PTS (priv->cached_buf);
|
||||
priv->last_backbuf_dur = GST_BUFFER_DURATION (priv->cached_buf);
|
||||
auto backbuf = priv->backbuf[priv->last_backbuf_idx]->backbuf;
|
||||
|
||||
GstD3D12FenceData *fence_data;
|
||||
gst_d3d12_fence_data_pool_acquire (priv->fence_data_pool, &fence_data);
|
||||
@ -1360,6 +1406,158 @@ gst_d3d12_swapchain_sink_redraw (GstD3D12SwapChainSink * self)
|
||||
}
|
||||
}
|
||||
|
||||
static GstSample *
|
||||
gst_d3d12_swapchain_sink_last_back_buffer (GstD3D12SwapChainSink * self,
|
||||
gboolean remove_borders)
|
||||
{
|
||||
auto priv = self->priv;
|
||||
|
||||
std::lock_guard < std::recursive_mutex > lk (priv->lock);
|
||||
if (!priv->swapchain || !priv->cl || !priv->ca_pool)
|
||||
return nullptr;
|
||||
|
||||
if (priv->viewport.w <= 0 || priv->viewport.h <= 0)
|
||||
return nullptr;
|
||||
|
||||
ComPtr < ID3D12Resource > backbuf;
|
||||
ComPtr < ID3D12Resource > dst_resource;
|
||||
auto hr = priv->swapchain->GetBuffer (priv->last_backbuf_idx,
|
||||
IID_PPV_ARGS (&backbuf));
|
||||
if (!gst_d3d12_result (hr, self->device))
|
||||
return nullptr;
|
||||
|
||||
auto device = gst_d3d12_device_get_device_handle (self->device);
|
||||
auto src_desc = GetDesc (backbuf);
|
||||
|
||||
UINT64 width;
|
||||
UINT height;
|
||||
if (remove_borders) {
|
||||
width = priv->viewport.w;
|
||||
height = priv->viewport.h;
|
||||
} else {
|
||||
width = src_desc.Width;
|
||||
height = src_desc.Height;
|
||||
}
|
||||
|
||||
auto dst_desc = CD3DX12_RESOURCE_DESC::Tex2D (DXGI_FORMAT_R8G8B8A8_UNORM,
|
||||
width, height, 1, 1, 1, 0,
|
||||
D3D12_RESOURCE_FLAG_ALLOW_RENDER_TARGET |
|
||||
D3D12_RESOURCE_FLAG_ALLOW_SIMULTANEOUS_ACCESS |
|
||||
D3D12_RESOURCE_FLAG_ALLOW_UNORDERED_ACCESS);
|
||||
auto heap_props = CD3DX12_HEAP_PROPERTIES (D3D12_HEAP_TYPE_DEFAULT);
|
||||
hr = device->CreateCommittedResource (&heap_props, D3D12_HEAP_FLAG_SHARED,
|
||||
&dst_desc, D3D12_RESOURCE_STATE_COMMON, nullptr,
|
||||
IID_PPV_ARGS (&dst_resource));
|
||||
if (!gst_d3d12_result (hr, self->device))
|
||||
return nullptr;
|
||||
|
||||
GstD3D12CmdAlloc *gst_ca;
|
||||
if (!gst_d3d12_cmd_alloc_pool_acquire (priv->ca_pool, &gst_ca))
|
||||
return nullptr;
|
||||
|
||||
auto ca = gst_d3d12_cmd_alloc_get_handle (gst_ca);
|
||||
hr = ca->Reset ();
|
||||
if (!gst_d3d12_result (hr, self->device)) {
|
||||
gst_d3d12_cmd_alloc_unref (gst_ca);
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
hr = priv->cl->Reset (ca, nullptr);
|
||||
if (!gst_d3d12_result (hr, self->device)) {
|
||||
gst_d3d12_cmd_alloc_unref (gst_ca);
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
auto barrier = CD3DX12_RESOURCE_BARRIER::Transition (backbuf.Get (),
|
||||
D3D12_RESOURCE_STATE_COMMON, D3D12_RESOURCE_STATE_COPY_SOURCE);
|
||||
priv->cl->ResourceBarrier (1, &barrier);
|
||||
|
||||
if (remove_borders) {
|
||||
D3D12_BOX src_box = { };
|
||||
src_box.left = priv->viewport.x;
|
||||
src_box.top = priv->viewport.y;
|
||||
src_box.right = priv->viewport.x + priv->viewport.w;
|
||||
src_box.bottom = priv->viewport.y + priv->viewport.h;
|
||||
src_box.front = 0;
|
||||
src_box.back = 1;
|
||||
|
||||
auto src_location = CD3DX12_TEXTURE_COPY_LOCATION (backbuf.Get ());
|
||||
auto dst_location = CD3DX12_TEXTURE_COPY_LOCATION (dst_resource.Get ());
|
||||
|
||||
priv->cl->CopyTextureRegion (&dst_location,
|
||||
0, 0, 0, &src_location, &src_box);
|
||||
} else {
|
||||
priv->cl->CopyResource (dst_resource.Get (), backbuf.Get ());
|
||||
}
|
||||
|
||||
barrier = CD3DX12_RESOURCE_BARRIER::Transition (backbuf.Get (),
|
||||
D3D12_RESOURCE_STATE_COPY_SOURCE, D3D12_RESOURCE_STATE_COMMON);
|
||||
priv->cl->ResourceBarrier (1, &barrier);
|
||||
|
||||
hr = priv->cl->Close ();
|
||||
if (!gst_d3d12_result (hr, self->device)) {
|
||||
gst_d3d12_cmd_alloc_unref (gst_ca);
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
ID3D12CommandList *cmd_list[] = { priv->cl.Get () };
|
||||
|
||||
hr = gst_d3d12_cmd_queue_execute_command_lists (priv->cq,
|
||||
1, cmd_list, &priv->fence_val);
|
||||
if (!gst_d3d12_result (hr, self->device)) {
|
||||
gst_d3d12_cmd_alloc_unref (gst_ca);
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
gst_d3d12_cmd_queue_set_notify (priv->cq, priv->fence_val,
|
||||
gst_ca, (GDestroyNotify) gst_d3d12_cmd_alloc_unref);
|
||||
|
||||
auto mem = gst_d3d12_allocator_alloc_wrapped (nullptr, self->device,
|
||||
dst_resource.Get (), 0, nullptr, nullptr);
|
||||
if (!mem)
|
||||
return nullptr;
|
||||
|
||||
GstVideoInfo info;
|
||||
gst_video_info_set_format (&info, GST_VIDEO_FORMAT_RGBA, (guint) width,
|
||||
height);
|
||||
info.fps_n = priv->info.fps_n;
|
||||
info.fps_d = priv->info.fps_d;
|
||||
|
||||
D3D12_PLACED_SUBRESOURCE_FOOTPRINT layout;
|
||||
device->GetCopyableFootprints (&dst_desc,
|
||||
0, 1, 0, &layout, nullptr, nullptr, nullptr);
|
||||
|
||||
gsize offset[4];
|
||||
gint stride[4];
|
||||
|
||||
offset[0] = 0;
|
||||
stride[0] = layout.Footprint.RowPitch;
|
||||
|
||||
auto buf = gst_buffer_new ();
|
||||
gst_buffer_append_memory (buf, mem);
|
||||
gst_buffer_add_video_meta_full (buf, GST_VIDEO_FRAME_FLAG_NONE,
|
||||
GST_VIDEO_INFO_FORMAT (&info), GST_VIDEO_INFO_WIDTH (&info),
|
||||
GST_VIDEO_INFO_HEIGHT (&info), GST_VIDEO_INFO_N_PLANES (&info),
|
||||
offset, stride);
|
||||
|
||||
GST_BUFFER_DTS (buf) = GST_CLOCK_TIME_NONE;
|
||||
GST_BUFFER_PTS (buf) = priv->last_backbuf_pts;
|
||||
GST_BUFFER_DURATION (buf) = priv->last_backbuf_dur;
|
||||
|
||||
auto fence = gst_d3d12_cmd_queue_get_fence_handle (priv->cq);
|
||||
gst_d3d12_buffer_set_fence (buf, fence, priv->fence_val, FALSE);
|
||||
|
||||
auto caps = gst_video_info_to_caps (&info);
|
||||
gst_caps_set_features_simple (caps,
|
||||
gst_caps_features_from_string (GST_CAPS_FEATURE_MEMORY_D3D12_MEMORY));
|
||||
|
||||
auto sample = gst_sample_new (buf, caps, &priv->segment, nullptr);
|
||||
gst_buffer_unref (buf);
|
||||
gst_caps_unref (caps);
|
||||
|
||||
return sample;
|
||||
}
|
||||
|
||||
static void
|
||||
gst_d3d12_swapchain_sink_resize (GstD3D12SwapChainSink * self, guint width,
|
||||
guint height)
|
||||
|
||||
@ -40,6 +40,7 @@ using namespace Microsoft::WRL;
|
||||
|
||||
static GMainLoop *loop_ = nullptr;
|
||||
static HWND hwnd_ = nullptr;
|
||||
static gchar *snapshot_location = nullptr;
|
||||
#define VIEW_WIDTH 640
|
||||
#define VIEW_HEIGHT 480
|
||||
#define REMAP_SIZE 1024
|
||||
@ -288,6 +289,35 @@ keyboard_cb (gchar input, gboolean is_ascii, AppData * app_data)
|
||||
if (state == GST_STATE_PAUSED)
|
||||
g_signal_emit_by_name (app_data->sink, "redraw");
|
||||
break;
|
||||
case 'c':
|
||||
case 'C':
|
||||
if (snapshot_location) {
|
||||
GstSample *sample = nullptr;
|
||||
GstSample *out_sample = nullptr;
|
||||
gboolean remove_borders = TRUE;
|
||||
g_signal_emit_by_name (app_data->sink, "last-rendered-sample",
|
||||
remove_borders, &sample);
|
||||
if (sample) {
|
||||
auto caps = gst_caps_new_simple ("image/jpeg", nullptr);
|
||||
out_sample = gst_video_convert_sample (sample, caps, 10 * GST_SECOND,
|
||||
nullptr);
|
||||
gst_caps_unref (caps);
|
||||
gst_sample_unref (sample);
|
||||
}
|
||||
|
||||
if (out_sample) {
|
||||
auto buf = gst_sample_get_buffer (out_sample);
|
||||
GstMapInfo map;
|
||||
gst_buffer_map (buf, &map, GST_MAP_READ);
|
||||
gst_println ("Writing snapshot to %s", snapshot_location);
|
||||
g_file_set_contents (snapshot_location, (gchar *) map.data,
|
||||
map.size, nullptr);
|
||||
|
||||
gst_buffer_unmap (buf, &map);
|
||||
gst_sample_unref (out_sample);
|
||||
}
|
||||
}
|
||||
break;
|
||||
case 'q':
|
||||
g_main_loop_quit (loop_);
|
||||
break;
|
||||
@ -472,6 +502,7 @@ print_keyboard_help (void)
|
||||
} key_controls[] = {
|
||||
{"m", "Toggle remap on/off"},
|
||||
{"space", "Toggle pause/play"},
|
||||
{"c", "Capture snapshot"},
|
||||
{"q", "Quit"},
|
||||
};
|
||||
|
||||
@ -503,6 +534,8 @@ main (int argc, char ** argv)
|
||||
gchar *uri = nullptr;
|
||||
GOptionEntry options[] = {
|
||||
{"uri", 0, 0, G_OPTION_ARG_STRING, &uri, "URI to play"},
|
||||
{"snapshot-location", 0, 0, G_OPTION_ARG_STRING, &snapshot_location,
|
||||
"JPEG file path for saving the snapshot image"},
|
||||
{nullptr}
|
||||
};
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user