From 586390b1baaf01c5d9e36b4f4db04336076dfa66 Mon Sep 17 00:00:00 2001 From: Seungha Yang Date: Thu, 26 Dec 2019 14:24:46 +0900 Subject: [PATCH] d3d11: Add h264 decoder element New decoder implementation based on dxva2 on d3d11 APIs. The DPB management implementation is taken from Chromium. --- sys/d3d11/gstd3d11_fwd.h | 7 + sys/d3d11/gstd3d11decoder.c | 1014 +++++++++++++++++++ sys/d3d11/gstd3d11decoder.h | 129 +++ sys/d3d11/gstd3d11h264dec.c | 1186 +++++++++++++++++++++++ sys/d3d11/gstd3d11h264dec.h | 87 ++ sys/d3d11/gsth264decoder.c | 1829 +++++++++++++++++++++++++++++++++++ sys/d3d11/gsth264decoder.h | 112 +++ sys/d3d11/gsth264picture.c | 510 ++++++++++ sys/d3d11/gsth264picture.h | 216 +++++ sys/d3d11/meson.build | 15 + sys/d3d11/plugin.c | 19 + 11 files changed, 5124 insertions(+) create mode 100644 sys/d3d11/gstd3d11decoder.c create mode 100644 sys/d3d11/gstd3d11decoder.h create mode 100644 sys/d3d11/gstd3d11h264dec.c create mode 100644 sys/d3d11/gstd3d11h264dec.h create mode 100644 sys/d3d11/gsth264decoder.c create mode 100644 sys/d3d11/gsth264decoder.h create mode 100644 sys/d3d11/gsth264picture.c create mode 100644 sys/d3d11/gsth264picture.h diff --git a/sys/d3d11/gstd3d11_fwd.h b/sys/d3d11/gstd3d11_fwd.h index 9485c206ef..f9d8837554 100644 --- a/sys/d3d11/gstd3d11_fwd.h +++ b/sys/d3d11/gstd3d11_fwd.h @@ -72,6 +72,13 @@ typedef struct _GstD3D11DownloadClass GstD3D11DownloadClass; typedef struct _GstD3D11ColorConvert GstD3D11ColorConvert; typedef struct _GstD3D11ColorConvertClass GstD3D11ColorConvertClass; +typedef struct _GstD3D11Decoder GstD3D11Decoder; +typedef struct _GstD3D11DecoderClass GstD3D11DecoderClass; +typedef struct _GstD3D11DecoderPrivate GstD3D11DecoderPrivate; + +typedef struct _GstD3D11H264Dec GstD3D11H264Dec; +typedef struct _GstD3D11H264DecClass GstD3D11H264DecClass; + G_END_DECLS #endif /* __GST_D3D11_FWD_H__ */ diff --git a/sys/d3d11/gstd3d11decoder.c b/sys/d3d11/gstd3d11decoder.c new file mode 100644 index 0000000000..a2bb891584 --- /dev/null +++ b/sys/d3d11/gstd3d11decoder.c @@ -0,0 +1,1014 @@ +/* GStreamer + * Copyright (C) 2019 Seungha Yang + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include "gstd3d11decoder.h" +#include "gstd3d11memory.h" +#include "gstd3d11bufferpool.h" +#include "gstd3d11device.h" +#include + +GST_DEBUG_CATEGORY (d3d11_decoder_debug); +#define GST_CAT_DEFAULT d3d11_decoder_debug + +enum +{ + PROP_0, + PROP_DEVICE, +}; + +struct _GstD3D11DecoderPrivate +{ + GstD3D11Device *device; + + ID3D11VideoDevice *video_device; + ID3D11VideoContext *video_context; + + ID3D11VideoDecoder *decoder; + + GstBufferPool *internal_pool; + + /* for staging */ + ID3D11Texture2D *staging; + D3D11_TEXTURE2D_DESC staging_desc; +}; + +#define OUTPUT_VIEW_QUARK _decoder_output_view_get() +static GQuark +_decoder_output_view_get (void) +{ + static gsize g_quark; + + if (g_once_init_enter (&g_quark)) { + gsize quark = + (gsize) g_quark_from_static_string ("GstD3D11DecoderOutputView"); + g_once_init_leave (&g_quark, quark); + } + return g_quark; +} + +static void gst_d3d11_decoder_constructed (GObject * object); +static void gst_d3d11_decoder_set_property (GObject * object, guint prop_id, + const GValue * value, GParamSpec * pspec); +static void gst_d3d11_decoder_get_property (GObject * object, guint prop_id, + GValue * value, GParamSpec * pspec); +static void gst_d3d11_decoder_dispose (GObject * obj); + +#define parent_class gst_d3d11_decoder_parent_class +G_DEFINE_TYPE_WITH_PRIVATE (GstD3D11Decoder, + gst_d3d11_decoder, GST_TYPE_OBJECT); + +static void +gst_d3d11_decoder_class_init (GstD3D11DecoderClass * klass) +{ + GObjectClass *gobject_class = G_OBJECT_CLASS (klass); + + gobject_class->constructed = gst_d3d11_decoder_constructed; + gobject_class->set_property = gst_d3d11_decoder_set_property; + gobject_class->get_property = gst_d3d11_decoder_get_property; + gobject_class->dispose = gst_d3d11_decoder_dispose; + + g_object_class_install_property (gobject_class, PROP_DEVICE, + g_param_spec_object ("device", "Device", + "D3D11 Devicd to use", GST_TYPE_D3D11_DEVICE, + G_PARAM_READWRITE | G_PARAM_CONSTRUCT_ONLY | G_PARAM_STATIC_STRINGS)); + + GST_DEBUG_CATEGORY_INIT (d3d11_decoder_debug, + "d3d11decoder", 0, "Direct3D11 Base Video Decoder object"); +} + +static void +gst_d3d11_decoder_init (GstD3D11Decoder * self) +{ + self->priv = gst_d3d11_decoder_get_instance_private (self); +} + +static void +gst_d3d11_decoder_constructed (GObject * object) +{ + GstD3D11Decoder *self = GST_D3D11_DECODER (object); + GstD3D11DecoderPrivate *priv = self->priv; + HRESULT hr; + ID3D11Device *device_handle; + ID3D11DeviceContext *device_context_handle; + + if (!priv->device) { + GST_ERROR_OBJECT (self, "No D3D11Device available"); + return; + } + + device_handle = gst_d3d11_device_get_device_handle (priv->device); + device_context_handle = + gst_d3d11_device_get_device_context_handle (priv->device); + + hr = ID3D11Device_QueryInterface (device_handle, &IID_ID3D11VideoDevice, + (void **) &priv->video_device); + + if (!gst_d3d11_result (hr, priv->device) || !priv->video_device) { + GST_ERROR_OBJECT (self, "Cannot create VideoDevice Object: 0x%x", + (guint) hr); + priv->video_device = NULL; + + return; + } + + hr = ID3D11DeviceContext_QueryInterface (device_context_handle, + &IID_ID3D11VideoContext, (void **) &priv->video_context); + + if (!gst_d3d11_result (hr, priv->device) || !priv->video_context) { + GST_ERROR_OBJECT (self, "Cannot create VideoContext Object: 0x%x", + (guint) hr); + priv->video_context = NULL; + + goto fail; + } + + return; + +fail: + if (priv->video_device) { + ID3D11VideoDevice_Release (priv->video_device); + priv->video_device = NULL; + } + + if (priv->video_context) { + ID3D11VideoContext_Release (priv->video_context); + priv->video_device = NULL; + } + + return; +} + +static void +gst_d3d11_decoder_set_property (GObject * object, guint prop_id, + const GValue * value, GParamSpec * pspec) +{ + GstD3D11Decoder *self = GST_D3D11_DECODER (object); + GstD3D11DecoderPrivate *priv = self->priv; + + switch (prop_id) { + case PROP_DEVICE: + priv->device = g_value_dup_object (value); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static void +gst_d3d11_decoder_get_property (GObject * object, guint prop_id, + GValue * value, GParamSpec * pspec) +{ + GstD3D11Decoder *self = GST_D3D11_DECODER (object); + GstD3D11DecoderPrivate *priv = self->priv; + + switch (prop_id) { + case PROP_DEVICE: + g_value_set_object (value, priv->device); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static gboolean +gst_d3d11_decoder_close (GstD3D11Decoder * self) +{ + GstD3D11DecoderPrivate *priv = self->priv; + + gst_d3d11_decoder_reset (self); + + if (priv->video_device) { + ID3D11VideoDevice_Release (priv->video_device); + priv->video_device = NULL; + } + + if (priv->video_context) { + ID3D11VideoContext_Release (priv->video_context); + priv->video_device = NULL; + } + + return TRUE; +} + +static void +gst_d3d11_decoder_reset_unlocked (GstD3D11Decoder * decoder) +{ + GstD3D11DecoderPrivate *priv; + + priv = decoder->priv; + gst_clear_object (&priv->internal_pool); + + if (priv->decoder) { + ID3D11VideoDecoder_Release (priv->decoder); + priv->decoder = NULL; + } + + if (priv->staging) { + ID3D11Texture2D_Release (priv->staging); + priv->staging = NULL; + } + + decoder->opened = FALSE; +} + +void +gst_d3d11_decoder_reset (GstD3D11Decoder * decoder) +{ + GstD3D11DecoderPrivate *priv; + + g_return_if_fail (GST_IS_D3D11_DECODER (decoder)); + + priv = decoder->priv; + gst_d3d11_device_lock (priv->device); + gst_d3d11_decoder_reset_unlocked (decoder); + gst_d3d11_device_unlock (priv->device); +} + +static void +gst_d3d11_decoder_dispose (GObject * obj) +{ + GstD3D11Decoder *self = GST_D3D11_DECODER (obj); + GstD3D11DecoderPrivate *priv = self->priv; + + if (priv->device) { + gst_d3d11_decoder_close (self); + gst_object_unref (priv->device); + priv->device = NULL; + } + + G_OBJECT_CLASS (parent_class)->dispose (obj); +} + +GstD3D11Decoder * +gst_d3d11_decoder_new (GstD3D11Device * device) +{ + GstD3D11Decoder *decoder; + + g_return_val_if_fail (GST_IS_D3D11_DEVICE (device), NULL); + + decoder = g_object_new (GST_TYPE_D3D11_DECODER, "device", device, NULL); + + if (decoder) + gst_object_ref_sink (decoder); + + return decoder; +} + +static void +gst_d3d11_decoder_output_view_free (GstD3D11DecoderOutputView * view) +{ + GST_DEBUG_OBJECT (view->device, "Free view %p", view); + + if (view->handle) { + gst_d3d11_device_lock (view->device); + ID3D11VideoDecoderOutputView_Release (view->handle); + gst_d3d11_device_unlock (view->device); + } + + gst_clear_object (&view->device); + g_free (view); +} + +static gboolean +gst_d3d11_decoder_prepare_output_view (GstD3D11Decoder * self, + GstBufferPool * pool, guint pool_size, const GUID * decoder_profile) +{ + GstD3D11DecoderPrivate *priv = self->priv; + GstBuffer **list = NULL; + D3D11_VIDEO_DECODER_OUTPUT_VIEW_DESC view_desc = { 0, }; + guint i; + + view_desc.DecodeProfile = *decoder_profile; + view_desc.ViewDimension = D3D11_VDOV_DIMENSION_TEXTURE2D; + + list = g_newa (GstBuffer *, pool_size); + memset (list, 0, sizeof (GstBuffer *) * pool_size); + + /* create output view here */ + for (i = 0; i < pool_size; i++) { + GstFlowReturn ret; + GstBuffer *buf = NULL; + GstD3D11Memory *mem; + GstD3D11DecoderOutputView *view; + ID3D11VideoDecoderOutputView *view_handle; + HRESULT hr; + ID3D11Texture2D *texture; + GstD3D11Allocator *allocator; + + ret = gst_buffer_pool_acquire_buffer (pool, &buf, NULL); + + if (ret != GST_FLOW_OK || !buf) { + GST_ERROR_OBJECT (self, "Could acquire buffer from pool"); + goto error; + } + + list[i] = buf; + mem = (GstD3D11Memory *) gst_buffer_peek_memory (buf, 0); + + allocator = GST_D3D11_ALLOCATOR (GST_MEMORY_CAST (mem)->allocator); + texture = allocator->texture; + + if (!texture) { + GST_ERROR_OBJECT (self, "D3D11 allocator does not have texture"); + goto error; + } + + view_desc.Texture2D.ArraySlice = mem->subresource_index; + GST_LOG_OBJECT (self, + "Create decoder output view with index %d", mem->subresource_index); + + hr = ID3D11VideoDevice_CreateVideoDecoderOutputView (priv->video_device, + (ID3D11Resource *) texture, &view_desc, &view_handle); + if (!gst_d3d11_result (hr, priv->device)) { + GST_ERROR_OBJECT (self, + "Could not create decoder output view, hr: 0x%x", (guint) hr); + goto error; + } + + view = g_new0 (GstD3D11DecoderOutputView, 1); + view->device = gst_object_ref (priv->device); + view->handle = view_handle; + view->view_id = mem->subresource_index; + + gst_mini_object_set_qdata (GST_MINI_OBJECT_CAST (mem), OUTPUT_VIEW_QUARK, + view, (GDestroyNotify) gst_d3d11_decoder_output_view_free); + } + + /* return buffers to pool */ + for (i = 0; i < pool_size; i++) { + gst_buffer_unref (list[i]); + } + + return TRUE; + +error: + if (list) { + for (i = 0; i < pool_size; i++) { + if (list[i]) + gst_buffer_unref (list[i]); + } + } + + return FALSE; +} + +/* Must be called from D3D11Device thread */ +static gboolean +gst_d3d11_decoder_prepare_output_view_pool (GstD3D11Decoder * self, + GstVideoInfo * info, guint pool_size, const GUID * decoder_profile) +{ + GstD3D11DecoderPrivate *priv = self->priv; + GstD3D11AllocationParams *alloc_params = NULL; + GstBufferPool *pool = NULL; + GstStructure *config = NULL; + GstCaps *caps = NULL; + + gst_clear_object (&priv->internal_pool); + + alloc_params = gst_d3d11_allocation_params_new (info, + GST_D3D11_ALLOCATION_FLAG_TEXTURE_ARRAY, D3D11_USAGE_DEFAULT, + D3D11_BIND_DECODER); + + if (!alloc_params) { + GST_ERROR_OBJECT (self, "Failed to create allocation param"); + goto error; + } + + alloc_params->desc[0].ArraySize = pool_size; + + pool = gst_d3d11_buffer_pool_new (priv->device); + if (!pool) { + GST_ERROR_OBJECT (self, "Failed to create buffer pool"); + goto error; + } + + /* Setup buffer pool */ + config = gst_buffer_pool_get_config (pool); + caps = gst_video_info_to_caps (info); + if (!caps) { + GST_ERROR_OBJECT (self, "Couldn't convert video info to caps"); + goto error; + } + + gst_buffer_pool_config_set_params (config, caps, GST_VIDEO_INFO_SIZE (info), + 0, pool_size); + gst_buffer_pool_config_set_d3d11_allocation_params (config, alloc_params); + gst_caps_unref (caps); + gst_d3d11_allocation_params_free (alloc_params); + gst_buffer_pool_config_add_option (config, GST_BUFFER_POOL_OPTION_VIDEO_META); + + if (!gst_buffer_pool_set_config (pool, config)) { + GST_ERROR_OBJECT (self, "Invalid pool config"); + goto error; + } + + if (!gst_buffer_pool_set_active (pool, TRUE)) { + GST_ERROR_OBJECT (self, "Couldn't activate pool"); + goto error; + } + + if (!gst_d3d11_decoder_prepare_output_view (self, + pool, pool_size, decoder_profile)) { + goto error; + } + + priv->internal_pool = pool; + + return TRUE; + +error: + if (alloc_params) + gst_d3d11_allocation_params_free (alloc_params); + if (pool) + gst_object_unref (pool); + if (caps) + gst_caps_unref (caps); + + return FALSE; +} + +gboolean +gst_d3d11_decoder_open (GstD3D11Decoder * decoder, GstD3D11Codec codec, + GstVideoInfo * info, guint pool_size, const GUID ** decoder_profiles, + guint profile_size) +{ + GstD3D11DecoderPrivate *priv; + const GstD3D11Format *d3d11_format; + HRESULT hr; + BOOL can_support = FALSE; + guint config_count; + D3D11_VIDEO_DECODER_CONFIG *config_list; + D3D11_VIDEO_DECODER_CONFIG *best_config = NULL; + D3D11_VIDEO_DECODER_DESC decoder_desc = { 0, }; + const GUID *selected_profile = NULL; + GUID *guid_list = NULL; + guint available_profile_count; + gint i, j; + + g_return_val_if_fail (GST_IS_D3D11_DECODER (decoder), FALSE); + g_return_val_if_fail (codec > GST_D3D11_CODEC_NONE, FALSE); + g_return_val_if_fail (codec < GST_D3D11_CODEC_LAST, FALSE); + g_return_val_if_fail (info != NULL, FALSE); + g_return_val_if_fail (pool_size > 0, FALSE); + g_return_val_if_fail (decoder_profiles != NULL, FALSE); + g_return_val_if_fail (profile_size > 0, FALSE); + + priv = decoder->priv; + decoder->opened = FALSE; + + d3d11_format = gst_d3d11_format_from_gst (GST_VIDEO_INFO_FORMAT (info)); + if (!d3d11_format || d3d11_format->dxgi_format == DXGI_FORMAT_UNKNOWN) { + GST_ERROR_OBJECT (decoder, "Could not determine dxgi format from %s", + gst_video_format_to_string (GST_VIDEO_INFO_FORMAT (info))); + return FALSE; + } + + gst_d3d11_device_lock (priv->device); + available_profile_count = + ID3D11VideoDevice_GetVideoDecoderProfileCount (priv->video_device); + + if (available_profile_count == 0) { + GST_ERROR_OBJECT (decoder, "No available decoder profile"); + goto error; + } + + GST_DEBUG_OBJECT (decoder, + "Have %u available decoder profiles", available_profile_count); + guid_list = g_alloca (sizeof (GUID) * available_profile_count); + + for (i = 0; i < available_profile_count; i++) { + hr = ID3D11VideoDevice_GetVideoDecoderProfile (priv->video_device, + i, &guid_list[i]); + if (!gst_d3d11_result (hr, priv->device)) { + GST_ERROR_OBJECT (decoder, "Failed to get %d th decoder profile", i); + goto error; + } + } + +#ifndef GST_DISABLE_GST_DEBUG + GST_LOG_OBJECT (decoder, "Supported decoder GUID"); + for (i = 0; i < available_profile_count; i++) { + const GUID *guid = &guid_list[i]; + + GST_LOG_OBJECT (decoder, + "\t { %8.8x-%4.4x-%4.4x-%2.2x%2.2x-%2.2x%2.2x%2.2x%2.2x%2.2x%2.2x }", + (guint) guid->Data1, (guint) guid->Data2, (guint) guid->Data3, + guid->Data4[0], guid->Data4[1], guid->Data4[2], guid->Data4[3], + guid->Data4[4], guid->Data4[5], guid->Data4[6], guid->Data4[7]); + } + + GST_LOG_OBJECT (decoder, "Requested decoder GUID"); + for (i = 0; i < profile_size; i++) { + const GUID *guid = decoder_profiles[i]; + + GST_LOG_OBJECT (decoder, + "\t { %8.8x-%4.4x-%4.4x-%2.2x%2.2x-%2.2x%2.2x%2.2x%2.2x%2.2x%2.2x }", + (guint) guid->Data1, (guint) guid->Data2, (guint) guid->Data3, + guid->Data4[0], guid->Data4[1], guid->Data4[2], guid->Data4[3], + guid->Data4[4], guid->Data4[5], guid->Data4[6], guid->Data4[7]); + } +#endif + + for (i = 0; i < profile_size; i++) { + for (j = 0; j < available_profile_count; j++) { + if (IsEqualGUID (decoder_profiles[i], &guid_list[j])) { + selected_profile = decoder_profiles[i]; + break; + } + } + } + + if (!selected_profile) { + GST_ERROR_OBJECT (decoder, "No supported decoder profile"); + goto error; + } else { + GST_DEBUG_OBJECT (decoder, + "Selected guid " + "{ %8.8x-%4.4x-%4.4x-%2.2x%2.2x-%2.2x%2.2x%2.2x%2.2x%2.2x%2.2x }", + (guint) selected_profile->Data1, (guint) selected_profile->Data2, + (guint) selected_profile->Data3, + selected_profile->Data4[0], selected_profile->Data4[1], + selected_profile->Data4[2], selected_profile->Data4[3], + selected_profile->Data4[4], selected_profile->Data4[5], + selected_profile->Data4[6], selected_profile->Data4[7]); + } + + hr = ID3D11VideoDevice_CheckVideoDecoderFormat (priv->video_device, + selected_profile, d3d11_format->dxgi_format, &can_support); + if (!gst_d3d11_result (hr, priv->device) || !can_support) { + GST_ERROR_OBJECT (decoder, + "VideoDevice could not support dxgi format %d, hr: 0x%x", + d3d11_format->dxgi_format, (guint) hr); + goto error; + } + + gst_d3d11_decoder_reset_unlocked (decoder); + + decoder_desc.SampleWidth = GST_VIDEO_INFO_WIDTH (info); + decoder_desc.SampleHeight = GST_VIDEO_INFO_HEIGHT (info); + decoder_desc.OutputFormat = d3d11_format->dxgi_format; + decoder_desc.Guid = *selected_profile; + + hr = ID3D11VideoDevice_GetVideoDecoderConfigCount (priv->video_device, + &decoder_desc, &config_count); + if (!gst_d3d11_result (hr, priv->device) || config_count == 0) { + GST_ERROR_OBJECT (decoder, "Could not get decoder config count, hr: 0x%x", + (guint) hr); + goto error; + } + + GST_DEBUG_OBJECT (decoder, "Total %d config available", config_count); + + config_list = g_alloca (sizeof (D3D11_VIDEO_DECODER_CONFIG) * config_count); + + for (i = 0; i < config_count; i++) { + hr = ID3D11VideoDevice_GetVideoDecoderConfig (priv->video_device, + &decoder_desc, i, &config_list[i]); + if (!gst_d3d11_result (hr, priv->device)) { + GST_ERROR_OBJECT (decoder, "Could not get decoder %dth config, hr: 0x%x", + i, (guint) hr); + goto error; + } + + /* FIXME: need support DXVA_Slice_H264_Long ?? */ + /* this config uses DXVA_Slice_H264_Short */ + if (codec == GST_D3D11_CODEC_H264 && config_list[i].ConfigBitstreamRaw == 2) { + best_config = &config_list[i]; + break; + } + } + + if (best_config == NULL) { + GST_ERROR_OBJECT (decoder, "Could not determine decoder config"); + goto error; + } + + if (!gst_d3d11_decoder_prepare_output_view_pool (decoder, + info, pool_size, selected_profile)) { + GST_ERROR_OBJECT (decoder, "Couldn't prepare output view pool"); + goto error; + } + + hr = ID3D11VideoDevice_CreateVideoDecoder (priv->video_device, + &decoder_desc, best_config, &priv->decoder); + if (!gst_d3d11_result (hr, priv->device) || !priv->decoder) { + GST_ERROR_OBJECT (decoder, + "Could not create decoder object, hr: 0x%x", (guint) hr); + goto error; + } + + GST_DEBUG_OBJECT (decoder, "Decoder object %p created", priv->decoder); + + /* create stage texture to copy out */ + memset (&priv->staging_desc, 0, sizeof (D3D11_TEXTURE2D_DESC)); + priv->staging_desc.Width = GST_VIDEO_INFO_WIDTH (info); + priv->staging_desc.Height = GST_VIDEO_INFO_HEIGHT (info); + priv->staging_desc.MipLevels = 1; + priv->staging_desc.Format = d3d11_format->dxgi_format; + priv->staging_desc.SampleDesc.Count = 1; + priv->staging_desc.ArraySize = 1; + priv->staging_desc.Usage = D3D11_USAGE_STAGING; + priv->staging_desc.CPUAccessFlags = D3D11_CPU_ACCESS_READ; + + priv->staging = gst_d3d11_device_create_texture (priv->device, + &priv->staging_desc, NULL); + if (!priv->staging) { + GST_ERROR_OBJECT (decoder, "Couldn't create staging texture"); + goto error; + } + + decoder->opened = TRUE; + gst_d3d11_device_unlock (priv->device); + + return TRUE; + +error: + gst_d3d11_device_unlock (priv->device); + + return FALSE; +} + +gboolean +gst_d3d11_decoder_begin_frame (GstD3D11Decoder * decoder, + GstD3D11DecoderOutputView * output_view, guint content_key_size, + gconstpointer content_key) +{ + GstD3D11DecoderPrivate *priv; + guint retry_count = 0; + HRESULT hr; + + g_return_val_if_fail (GST_IS_D3D11_DECODER (decoder), FALSE); + g_return_val_if_fail (output_view != NULL, FALSE); + g_return_val_if_fail (output_view->handle != NULL, FALSE); + + priv = decoder->priv; + + do { + GST_LOG_OBJECT (decoder, "Try begin frame, retry count %d", retry_count); + gst_d3d11_device_lock (priv->device); + hr = ID3D11VideoContext_DecoderBeginFrame (priv->video_context, + priv->decoder, output_view->handle, content_key_size, content_key); + gst_d3d11_device_unlock (priv->device); + + if (gst_d3d11_result (hr, priv->device)) { + GST_LOG_OBJECT (decoder, "Success with retry %d", retry_count); + break; + } else if (hr == E_PENDING && retry_count < 50) { + GST_LOG_OBJECT (decoder, "GPU busy, try again"); + + /* HACK: no better idea other than sleep... + * 1ms waiting like msdkdec */ + g_usleep (1000); + } else { + break; + } + + retry_count++; + } while (TRUE); + + if (!gst_d3d11_result (hr, priv->device)) { + GST_ERROR_OBJECT (decoder, "Failed to begin frame, hr: 0x%x", (guint) hr); + return FALSE; + } + + return TRUE; +} + +gboolean +gst_d3d11_decoder_end_frame (GstD3D11Decoder * decoder) +{ + GstD3D11DecoderPrivate *priv; + HRESULT hr; + + g_return_val_if_fail (GST_IS_D3D11_DECODER (decoder), FALSE); + + priv = decoder->priv; + + gst_d3d11_device_lock (priv->device); + hr = ID3D11VideoContext_DecoderEndFrame (priv->video_context, priv->decoder); + gst_d3d11_device_unlock (priv->device); + + if (!gst_d3d11_result (hr, priv->device)) { + GST_WARNING_OBJECT (decoder, "EndFrame failed, hr: 0x%x", (guint) hr); + return FALSE; + } + + return TRUE; +} + +gboolean +gst_d3d11_decoder_get_decoder_buffer (GstD3D11Decoder * decoder, + D3D11_VIDEO_DECODER_BUFFER_TYPE type, guint * buffer_size, + gpointer * buffer) +{ + GstD3D11DecoderPrivate *priv; + UINT size; + void *decoder_buffer; + HRESULT hr; + + g_return_val_if_fail (GST_IS_D3D11_DECODER (decoder), FALSE); + + priv = decoder->priv; + + gst_d3d11_device_lock (priv->device); + hr = ID3D11VideoContext_GetDecoderBuffer (priv->video_context, + priv->decoder, type, &size, &decoder_buffer); + gst_d3d11_device_unlock (priv->device); + + if (!gst_d3d11_result (hr, priv->device)) { + GST_WARNING_OBJECT (decoder, "Getting buffer type %d error, hr: 0x%x", + type, (guint) hr); + return FALSE; + } + + *buffer_size = size; + *buffer = decoder_buffer; + + return TRUE; +} + +gboolean +gst_d3d11_decoder_release_decoder_buffer (GstD3D11Decoder * decoder, + D3D11_VIDEO_DECODER_BUFFER_TYPE type) +{ + GstD3D11DecoderPrivate *priv; + HRESULT hr; + + g_return_val_if_fail (GST_IS_D3D11_DECODER (decoder), FALSE); + + priv = decoder->priv; + + gst_d3d11_device_lock (priv->device); + hr = ID3D11VideoContext_ReleaseDecoderBuffer (priv->video_context, + priv->decoder, type); + gst_d3d11_device_unlock (priv->device); + + if (!gst_d3d11_result (hr, priv->device)) { + GST_WARNING_OBJECT (decoder, "ReleaseDecoderBuffer failed, hr: 0x%x", + (guint) hr); + return FALSE; + } + + return TRUE; +} + +gboolean +gst_d3d11_decoder_submit_decoder_buffers (GstD3D11Decoder * decoder, + guint buffer_count, const D3D11_VIDEO_DECODER_BUFFER_DESC * buffers) +{ + GstD3D11DecoderPrivate *priv; + HRESULT hr; + + g_return_val_if_fail (GST_IS_D3D11_DECODER (decoder), FALSE); + + priv = decoder->priv; + + gst_d3d11_device_lock (priv->device); + hr = ID3D11VideoContext_SubmitDecoderBuffers (priv->video_context, + priv->decoder, buffer_count, buffers); + gst_d3d11_device_unlock (priv->device); + + if (!gst_d3d11_result (hr, priv->device)) { + GST_WARNING_OBJECT (decoder, "SubmitDecoderBuffers failed, hr: 0x%x", + (guint) hr); + return FALSE; + } + + return TRUE; +} + +GstBuffer * +gst_d3d11_decoder_get_output_view_buffer (GstD3D11Decoder * decoder) +{ + GstD3D11DecoderPrivate *priv; + GstBuffer *buf = NULL; + GstFlowReturn ret; + + g_return_val_if_fail (GST_IS_D3D11_DECODER (decoder), FALSE); + + priv = decoder->priv; + + ret = gst_buffer_pool_acquire_buffer (priv->internal_pool, &buf, NULL); + + if (ret != GST_FLOW_OK || !buf) { + GST_ERROR_OBJECT (decoder, "Couldn't get buffer from pool, ret %s", + gst_flow_get_name (ret)); + return FALSE; + } + + return buf; +} + +GstD3D11DecoderOutputView * +gst_d3d11_decoder_get_output_view_from_buffer (GstD3D11Decoder * decoder, + GstBuffer * buffer) +{ + GstMemory *mem; + GstD3D11DecoderOutputView *view; + + g_return_val_if_fail (GST_IS_D3D11_DECODER (decoder), NULL); + g_return_val_if_fail (GST_IS_BUFFER (buffer), NULL); + + mem = gst_buffer_peek_memory (buffer, 0); + if (!gst_is_d3d11_memory (mem)) { + GST_WARNING_OBJECT (decoder, "nemory is not d3d11 memory"); + return NULL; + } + + view = (GstD3D11DecoderOutputView *) + gst_mini_object_get_qdata (GST_MINI_OBJECT_CAST (mem), OUTPUT_VIEW_QUARK); + + if (!view) { + GST_WARNING_OBJECT (decoder, "memory does not have output view"); + } + + return view; +} + +guint +gst_d3d11_decoder_get_output_view_index (GstD3D11Decoder * decoder, + ID3D11VideoDecoderOutputView * view_handle) +{ + D3D11_VIDEO_DECODER_OUTPUT_VIEW_DESC view_desc; + + g_return_val_if_fail (GST_IS_D3D11_DECODER (decoder), 0xff); + g_return_val_if_fail (view_handle != NULL, 0xff); + + ID3D11VideoDecoderOutputView_GetDesc (view_handle, &view_desc); + + return view_desc.Texture2D.ArraySlice; +} + +static gboolean +copy_to_system (GstD3D11Decoder * self, GstVideoInfo * info, + GstBuffer * decoder_buffer, GstBuffer * output) +{ + GstD3D11DecoderPrivate *priv = self->priv; + D3D11_TEXTURE2D_DESC *desc = &priv->staging_desc; + GstVideoFrame out_frame; + gint i; + GstD3D11Memory *in_mem; + D3D11_MAPPED_SUBRESOURCE map; + gsize offset[GST_VIDEO_MAX_PLANES]; + gint stride[GST_VIDEO_MAX_PLANES]; + gsize dummy; + HRESULT hr; + ID3D11DeviceContext *device_context = + gst_d3d11_device_get_device_context_handle (priv->device); + + if (!gst_video_frame_map (&out_frame, info, output, GST_MAP_WRITE)) { + GST_ERROR_OBJECT (self, "Couldn't map output buffer"); + return FALSE; + } + + in_mem = (GstD3D11Memory *) gst_buffer_peek_memory (decoder_buffer, 0); + + gst_d3d11_device_lock (priv->device); + ID3D11DeviceContext_CopySubresourceRegion (device_context, + (ID3D11Resource *) priv->staging, 0, 0, 0, 0, + (ID3D11Resource *) in_mem->texture, in_mem->subresource_index, NULL); + + hr = ID3D11DeviceContext_Map (device_context, + (ID3D11Resource *) priv->staging, 0, D3D11_MAP_READ, 0, &map); + + if (!gst_d3d11_result (hr, priv->device)) { + GST_ERROR_OBJECT (self, "Failed to map, hr: 0x%x", (guint) hr); + gst_d3d11_device_unlock (priv->device); + return FALSE; + } + + gst_d3d11_dxgi_format_get_size (desc->Format, desc->Width, desc->Height, + map.RowPitch, offset, stride, &dummy); + + for (i = 0; i < GST_VIDEO_FRAME_N_PLANES (&out_frame); i++) { + guint8 *src, *dst; + gint j; + gint width; + + src = (guint8 *) map.pData + offset[i]; + dst = GST_VIDEO_FRAME_PLANE_DATA (&out_frame, i); + width = GST_VIDEO_FRAME_COMP_WIDTH (&out_frame, i) * + GST_VIDEO_FRAME_COMP_PSTRIDE (&out_frame, i); + + for (j = 0; j < GST_VIDEO_FRAME_COMP_HEIGHT (&out_frame, i); j++) { + memcpy (dst, src, width); + dst += GST_VIDEO_FRAME_PLANE_STRIDE (&out_frame, i); + src += map.RowPitch; + } + } + + gst_video_frame_unmap (&out_frame); + ID3D11DeviceContext_Unmap (device_context, (ID3D11Resource *) priv->staging, + 0); + gst_d3d11_device_unlock (priv->device); + + return TRUE; +} + +static gboolean +copy_to_d3d11 (GstD3D11Decoder * self, GstVideoInfo * info, + GstBuffer * decoder_buffer, GstBuffer * output) +{ + GstD3D11DecoderPrivate *priv = self->priv; + gint i; + ID3D11DeviceContext *device_context = + gst_d3d11_device_get_device_context_handle (priv->device); + + gst_d3d11_device_lock (priv->device); + for (i = 0; i < gst_buffer_n_memory (output); i++) { + GstD3D11Memory *in_mem; + GstD3D11Memory *out_mem; + D3D11_BOX src_box; + D3D11_TEXTURE2D_DESC desc; + + in_mem = (GstD3D11Memory *) gst_buffer_peek_memory (decoder_buffer, i); + out_mem = (GstD3D11Memory *) gst_buffer_peek_memory (output, i); + + ID3D11Texture2D_GetDesc (out_mem->texture, &desc); + + src_box.left = 0; + src_box.top = 0; + src_box.front = 0; + src_box.back = 1; + src_box.right = desc.Width; + src_box.bottom = desc.Height; + + ID3D11DeviceContext_CopySubresourceRegion (device_context, + (ID3D11Resource *) out_mem->texture, + out_mem->subresource_index, 0, 0, 0, + (ID3D11Resource *) in_mem->texture, in_mem->subresource_index, + &src_box); + + GST_MINI_OBJECT_FLAG_SET (out_mem, GST_D3D11_MEMORY_TRANSFER_NEED_DOWNLOAD); + } + gst_d3d11_device_unlock (priv->device); + + return TRUE; +} + +gboolean +gst_d3d11_decoder_copy_decoder_buffer (GstD3D11Decoder * decoder, + GstVideoInfo * info, GstBuffer * decoder_buffer, GstBuffer * output) +{ + GstD3D11DecoderPrivate *priv; + gboolean can_device_copy = TRUE; + + g_return_val_if_fail (GST_IS_D3D11_DECODER (decoder), FALSE); + g_return_val_if_fail (GST_IS_BUFFER (decoder_buffer), FALSE); + g_return_val_if_fail (GST_IS_BUFFER (output), FALSE); + + priv = decoder->priv; + + if (gst_buffer_n_memory (decoder_buffer) == gst_buffer_n_memory (output)) { + gint i; + + for (i = 0; i < gst_buffer_n_memory (output); i++) { + GstMemory *mem; + GstD3D11Memory *dmem; + + mem = gst_buffer_peek_memory (output, i); + + if (!gst_is_d3d11_memory (mem)) { + can_device_copy = FALSE; + break; + } + + dmem = (GstD3D11Memory *) mem; + + if (dmem->device != priv->device) { + can_device_copy = FALSE; + break; + } + } + } else { + can_device_copy = FALSE; + } + + if (can_device_copy) { + return copy_to_d3d11 (decoder, info, decoder_buffer, output); + } + + return copy_to_system (decoder, info, decoder_buffer, output); +} diff --git a/sys/d3d11/gstd3d11decoder.h b/sys/d3d11/gstd3d11decoder.h new file mode 100644 index 0000000000..41169aaf10 --- /dev/null +++ b/sys/d3d11/gstd3d11decoder.h @@ -0,0 +1,129 @@ +/* GStreamer + * Copyright (C) 2019 Seungha Yang + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifndef __GST_D3D11_DECODER_H__ +#define __GST_D3D11_DECODER_H__ + +#include +#include +#include +#include +#include +#include "gstd3d11_fwd.h" +#include "gstd3d11device.h" +#include "gstd3d11utils.h" + +G_BEGIN_DECLS + +#define GST_TYPE_D3D11_DECODER \ + (gst_d3d11_decoder_get_type()) +#define GST_D3D11_DECODER(obj) \ + (G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_D3D11_DECODER,GstD3D11Decoder)) +#define GST_D3D11_DECODER_CLASS(klass) \ + (G_TYPE_CHECK_CLASS_CAST((klass),GST_TYPE_D3D11_DECODER,GstD3D11DecoderClass)) +#define GST_D3D11_DECODER_GET_CLASS(obj) \ + (G_TYPE_INSTANCE_GET_CLASS((obj),GST_TYPE_D3D11_DECODER,GstD3D11DecoderClass)) +#define GST_IS_D3D11_DECODER(obj) \ + (G_TYPE_CHECK_INSTANCE_TYPE((obj),GST_TYPE_D3D11_DECODER)) +#define GST_IS_D3D11_DECODER_CLASS(klass) \ + (G_TYPE_CHECK_CLASS_TYPE((klass),GST_TYPE_D3D11_DECODER)) + +typedef struct _GstD3D11DecoderOutputView GstD3D11DecoderOutputView; + +struct _GstD3D11DecoderOutputView +{ + GstD3D11Device *device; + ID3D11VideoDecoderOutputView *handle; + guint view_id; +}; + +typedef enum +{ + GST_D3D11_CODEC_NONE, + GST_D3D11_CODEC_H264, + + /* the last of supported codec */ + GST_D3D11_CODEC_LAST +} GstD3D11Codec; + +struct _GstD3D11Decoder +{ + GstObject parent; + + /* TRUE if decoder was successfully opened ever */ + gboolean opened; + + /*< private >*/ + GstD3D11DecoderPrivate *priv; + gpointer padding[GST_PADDING_LARGE]; +}; + +struct _GstD3D11DecoderClass +{ + GstObjectClass parent_class; +}; + +GType gst_d3d11_decoder_get_type (void); + +GstD3D11Decoder * gst_d3d11_decoder_new (GstD3D11Device * device); + +gboolean gst_d3d11_decoder_open (GstD3D11Decoder * decoder, + GstD3D11Codec codec, + GstVideoInfo * info, + guint pool_size, + const GUID ** decoder_profiles, + guint profile_size); + +void gst_d3d11_decoder_reset (GstD3D11Decoder * decoder); + +gboolean gst_d3d11_decoder_begin_frame (GstD3D11Decoder * decoder, + GstD3D11DecoderOutputView * output_view, + guint content_key_size, + gconstpointer content_key); + +gboolean gst_d3d11_decoder_end_frame (GstD3D11Decoder * decoder); + +gboolean gst_d3d11_decoder_get_decoder_buffer (GstD3D11Decoder * decoder, + D3D11_VIDEO_DECODER_BUFFER_TYPE type, + guint * buffer_size, + gpointer * buffer); + +gboolean gst_d3d11_decoder_release_decoder_buffer (GstD3D11Decoder * decoder, + D3D11_VIDEO_DECODER_BUFFER_TYPE type); + +gboolean gst_d3d11_decoder_submit_decoder_buffers (GstD3D11Decoder * decoder, + guint buffer_count, + const D3D11_VIDEO_DECODER_BUFFER_DESC * buffers); + +GstBuffer * gst_d3d11_decoder_get_output_view_buffer (GstD3D11Decoder * decoder); + +GstD3D11DecoderOutputView * gst_d3d11_decoder_get_output_view_from_buffer (GstD3D11Decoder * decoder, + GstBuffer * buffer); + +guint gst_d3d11_decoder_get_output_view_index (GstD3D11Decoder * decoder, + ID3D11VideoDecoderOutputView * view_handle); + +gboolean gst_d3d11_decoder_copy_decoder_buffer (GstD3D11Decoder * decoder, + GstVideoInfo * info, + GstBuffer * decoder_buffer, + GstBuffer * output); + +G_END_DECLS + +#endif /* __GST_D3D11_DECODER_H__ */ diff --git a/sys/d3d11/gstd3d11h264dec.c b/sys/d3d11/gstd3d11h264dec.c new file mode 100644 index 0000000000..20af28badd --- /dev/null +++ b/sys/d3d11/gstd3d11h264dec.c @@ -0,0 +1,1186 @@ +/* GStreamer + * Copyright (C) 2019 Seungha Yang + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + * NOTE: some of implementations are copied/modified from Chromium code + * + * Copyright 2015 The Chromium Authors. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include "gstd3d11h264dec.h" +#include "gstd3d11memory.h" +#include "gstd3d11bufferpool.h" +#include + +GST_DEBUG_CATEGORY_EXTERN (gst_d3d11_h264_dec_debug); +#define GST_CAT_DEFAULT gst_d3d11_h264_dec_debug + +enum +{ + PROP_0, + PROP_ADAPTER +}; + +#define DEFAULT_ADAPTER -1 + +/* copied from d3d11.h since mingw header doesn't define them */ +DEFINE_GUID (GST_GUID_D3D11_DECODER_PROFILE_H264_IDCT_FGT, 0x1b81be67, 0xa0c7, + 0x11d3, 0xb9, 0x84, 0x00, 0xc0, 0x4f, 0x2e, 0x73, 0xc5); +DEFINE_GUID (GST_GUID_D3D11_DECODER_PROFILE_H264_VLD_NOFGT, 0x1b81be68, 0xa0c7, + 0x11d3, 0xb9, 0x84, 0x00, 0xc0, 0x4f, 0x2e, 0x73, 0xc5); +DEFINE_GUID (GST_GUID_D3D11_DECODER_PROFILE_H264_VLD_FGT, 0x1b81be69, 0xa0c7, + 0x11d3, 0xb9, 0x84, 0x00, 0xc0, 0x4f, 0x2e, 0x73, 0xc5); + +/* worst case 16 (non-interlaced) + 4 margin */ +#define NUM_OUTPUT_VIEW 20 + +static GstStaticPadTemplate sink_template = +GST_STATIC_PAD_TEMPLATE (GST_VIDEO_DECODER_SINK_NAME, + GST_PAD_SINK, GST_PAD_ALWAYS, + GST_STATIC_CAPS ("video/x-h264, " + "stream-format=(string) { avc, avc3, byte-stream }, " + "alignment=(string) au, profile = (string) { high, main }") + ); + +static GstStaticPadTemplate src_template = + GST_STATIC_PAD_TEMPLATE (GST_VIDEO_DECODER_SRC_NAME, + GST_PAD_SRC, GST_PAD_ALWAYS, + GST_STATIC_CAPS (GST_VIDEO_CAPS_MAKE_WITH_FEATURES + (GST_CAPS_FEATURE_MEMORY_D3D11_MEMORY, "{ NV12, P010_10LE }") "; " + GST_VIDEO_CAPS_MAKE ("{ NV12, P010_10LE }"))); + +#define parent_class gst_d3d11_h264_dec_parent_class +G_DEFINE_TYPE (GstD3D11H264Dec, gst_d3d11_h264_dec, GST_TYPE_H264_DECODER); + +static void gst_d3d11_h264_dec_set_property (GObject * object, + guint prop_id, const GValue * value, GParamSpec * pspec); +static void gst_d3d11_h264_dec_get_property (GObject * object, + guint prop_id, GValue * value, GParamSpec * pspec); +static void gst_d3d11_h264_dec_dispose (GObject * object); +static void gst_d3d11_h264_dec_set_context (GstElement * element, + GstContext * context); + +static gboolean gst_d3d11_h264_dec_open (GstVideoDecoder * decoder); +static gboolean gst_d3d11_h264_dec_close (GstVideoDecoder * decoder); +static gboolean gst_d3d11_h264_dec_start (GstVideoDecoder * decoder); +static gboolean gst_d3d11_h264_dec_stop (GstVideoDecoder * decoder); +static GstFlowReturn gst_d3d11_h264_dec_handle_frame (GstVideoDecoder * + decoder, GstVideoCodecFrame * frame); +static gboolean gst_d3d11_h264_dec_negotiate (GstVideoDecoder * decoder); +static gboolean gst_d3d11_h264_dec_decide_allocation (GstVideoDecoder * + decoder, GstQuery * query); +static gboolean gst_d3d11_h264_dec_src_query (GstVideoDecoder * decoder, + GstQuery * query); + +/* GstH264Decoder */ +static gboolean gst_d3d11_h264_dec_new_sequence (GstH264Decoder * decoder, + const GstH264SPS * sps); +static gboolean gst_d3d11_h264_dec_new_picture (GstH264Decoder * decoder, + GstH264Picture * picture); +static GstFlowReturn gst_d3d11_h264_dec_output_picture (GstH264Decoder * + decoder, GstH264Picture * picture); +static gboolean gst_d3d11_h264_dec_start_picture (GstH264Decoder * decoder, + GstH264Picture * picture, GstH264Slice * slice, GstH264Dpb * dpb); +static gboolean gst_d3d11_h264_dec_decode_slice (GstH264Decoder * decoder, + GstH264Picture * picture, GstH264Slice * slice); +static gboolean gst_d3d11_h264_dec_end_picture (GstH264Decoder * decoder, + GstH264Picture * picture); + +static void +gst_d3d11_h264_dec_class_init (GstD3D11H264DecClass * klass) +{ + GObjectClass *gobject_class = G_OBJECT_CLASS (klass); + GstElementClass *element_class = GST_ELEMENT_CLASS (klass); + GstVideoDecoderClass *decoder_class = GST_VIDEO_DECODER_CLASS (klass); + GstH264DecoderClass *h264decoder_class = GST_H264_DECODER_CLASS (klass); + + gobject_class->set_property = gst_d3d11_h264_dec_set_property; + gobject_class->get_property = gst_d3d11_h264_dec_get_property; + gobject_class->dispose = gst_d3d11_h264_dec_dispose; + + g_object_class_install_property (gobject_class, PROP_ADAPTER, + g_param_spec_int ("adapter", "Adapter", + "Adapter index for creating device (-1 for default)", + -1, G_MAXINT32, DEFAULT_ADAPTER, + G_PARAM_READWRITE | GST_PARAM_MUTABLE_READY | + G_PARAM_STATIC_STRINGS)); + + element_class->set_context = + GST_DEBUG_FUNCPTR (gst_d3d11_h264_dec_set_context); + + gst_element_class_set_static_metadata (element_class, + "Direct3D11 H.264 Video Decoder", + "Codec/Decoder/Video/Hardware", + "A Direct3D11 based H.264 video decoder", + "Seungha Yang "); + + gst_element_class_add_static_pad_template (element_class, &sink_template); + gst_element_class_add_static_pad_template (element_class, &src_template); + + decoder_class->open = GST_DEBUG_FUNCPTR (gst_d3d11_h264_dec_open); + decoder_class->close = GST_DEBUG_FUNCPTR (gst_d3d11_h264_dec_close); + decoder_class->start = GST_DEBUG_FUNCPTR (gst_d3d11_h264_dec_start); + decoder_class->stop = GST_DEBUG_FUNCPTR (gst_d3d11_h264_dec_stop); + decoder_class->handle_frame = + GST_DEBUG_FUNCPTR (gst_d3d11_h264_dec_handle_frame); + decoder_class->negotiate = GST_DEBUG_FUNCPTR (gst_d3d11_h264_dec_negotiate); + decoder_class->decide_allocation = + GST_DEBUG_FUNCPTR (gst_d3d11_h264_dec_decide_allocation); + decoder_class->src_query = GST_DEBUG_FUNCPTR (gst_d3d11_h264_dec_src_query); + + h264decoder_class->new_sequence = + GST_DEBUG_FUNCPTR (gst_d3d11_h264_dec_new_sequence); + h264decoder_class->new_picture = + GST_DEBUG_FUNCPTR (gst_d3d11_h264_dec_new_picture); + h264decoder_class->output_picture = + GST_DEBUG_FUNCPTR (gst_d3d11_h264_dec_output_picture); + h264decoder_class->start_picture = + GST_DEBUG_FUNCPTR (gst_d3d11_h264_dec_start_picture); + h264decoder_class->decode_slice = + GST_DEBUG_FUNCPTR (gst_d3d11_h264_dec_decode_slice); + h264decoder_class->end_picture = + GST_DEBUG_FUNCPTR (gst_d3d11_h264_dec_end_picture); +} + +static void +gst_d3d11_h264_dec_init (GstD3D11H264Dec * self) +{ + self->slice_list = g_array_new (FALSE, TRUE, sizeof (DXVA_Slice_H264_Short)); + self->adapter = DEFAULT_ADAPTER; +} + +static void +gst_d3d11_h264_dec_set_property (GObject * object, guint prop_id, + const GValue * value, GParamSpec * pspec) +{ + GstD3D11H264Dec *self = GST_D3D11_H264_DEC (object); + + switch (prop_id) { + case PROP_ADAPTER: + self->adapter = g_value_get_int (value); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static void +gst_d3d11_h264_dec_get_property (GObject * object, guint prop_id, + GValue * value, GParamSpec * pspec) +{ + GstD3D11H264Dec *self = GST_D3D11_H264_DEC (object); + + switch (prop_id) { + case PROP_ADAPTER: + g_value_set_int (value, self->adapter); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static void +gst_d3d11_h264_dec_dispose (GObject * object) +{ + GstD3D11H264Dec *self = GST_D3D11_H264_DEC (object); + + if (self->slice_list) { + g_array_unref (self->slice_list); + self->slice_list = NULL; + } + + G_OBJECT_CLASS (parent_class)->dispose (object); +} + +static void +gst_d3d11_h264_dec_set_context (GstElement * element, GstContext * context) +{ + GstD3D11H264Dec *self = GST_D3D11_H264_DEC (element); + + gst_d3d11_handle_set_context (element, context, self->adapter, &self->device); + + GST_ELEMENT_CLASS (parent_class)->set_context (element, context); +} + +static gboolean +gst_d3d11_h264_dec_open (GstVideoDecoder * decoder) +{ + GstD3D11H264Dec *self = GST_D3D11_H264_DEC (decoder); + + if (!gst_d3d11_ensure_element_data (GST_ELEMENT_CAST (self), self->adapter, + &self->device)) { + GST_ERROR_OBJECT (self, "Cannot create d3d11device"); + return FALSE; + } + + self->d3d11_decoder = gst_d3d11_decoder_new (self->device); + + if (!self->d3d11_decoder) { + GST_ERROR_OBJECT (self, "Cannot create d3d11 decoder"); + gst_clear_object (&self->device); + return FALSE; + } + + return TRUE; +} + +static gboolean +gst_d3d11_h264_dec_close (GstVideoDecoder * decoder) +{ + GstD3D11H264Dec *self = GST_D3D11_H264_DEC (decoder); + + gst_clear_object (&self->d3d11_decoder); + gst_clear_object (&self->device); + + return TRUE; +} + +static gboolean +gst_d3d11_h264_dec_start (GstVideoDecoder * decoder) +{ + return GST_VIDEO_DECODER_CLASS (parent_class)->start (decoder); +} + +static gboolean +gst_d3d11_h264_dec_stop (GstVideoDecoder * decoder) +{ + GstD3D11H264Dec *self = GST_D3D11_H264_DEC (decoder); + + gst_h264_picture_replace (&self->current_picture, NULL); + if (self->output_state) + gst_video_codec_state_unref (self->output_state); + self->output_state = NULL; + + return GST_VIDEO_DECODER_CLASS (parent_class)->stop (decoder); +} + +static GstFlowReturn +gst_d3d11_h264_dec_handle_frame (GstVideoDecoder * decoder, + GstVideoCodecFrame * frame) +{ + GstD3D11H264Dec *self = GST_D3D11_H264_DEC (decoder); + GstBuffer *in_buf = frame->input_buffer; + + GST_LOG_OBJECT (self, + "handle frame, PTS: %" GST_TIME_FORMAT ", DTS: %" + GST_TIME_FORMAT, GST_TIME_ARGS (GST_BUFFER_PTS (in_buf)), + GST_TIME_ARGS (GST_BUFFER_DTS (in_buf))); + + if (!self->current_picture) { + GST_ERROR_OBJECT (self, "No current picture"); + gst_video_decoder_drop_frame (decoder, frame); + + return GST_FLOW_ERROR; + } + + gst_video_codec_frame_set_user_data (frame, + self->current_picture, (GDestroyNotify) gst_h264_picture_unref); + self->current_picture = NULL; + + gst_video_codec_frame_unref (frame); + + return GST_FLOW_OK; +} + +static gboolean +gst_d3d11_h264_dec_negotiate (GstVideoDecoder * decoder) +{ + GstD3D11H264Dec *self = GST_D3D11_H264_DEC (decoder); + GstH264Decoder *h264dec = GST_H264_DECODER (decoder); + GstCaps *peer_caps; + + GST_DEBUG_OBJECT (self, "negotiate"); + + if (self->output_state) + gst_video_codec_state_unref (self->output_state); + + self->output_state = + gst_video_decoder_set_output_state (GST_VIDEO_DECODER (self), + self->out_format, self->width, self->height, h264dec->input_state); + + self->output_state->caps = gst_video_info_to_caps (&self->output_state->info); + + peer_caps = gst_pad_get_allowed_caps (GST_VIDEO_DECODER_SRC_PAD (self)); + GST_DEBUG_OBJECT (self, "Allowed caps %" GST_PTR_FORMAT, peer_caps); + + self->use_d3d11_output = FALSE; + + if (!peer_caps || gst_caps_is_any (peer_caps)) { + GST_DEBUG_OBJECT (self, + "cannot determine output format, use system memory"); + } else { + GstCapsFeatures *features; + guint size = gst_caps_get_size (peer_caps); + guint i; + + for (i = 0; i < size; i++) { + features = gst_caps_get_features (peer_caps, i); + if (features && gst_caps_features_contains (features, + GST_CAPS_FEATURE_MEMORY_D3D11_MEMORY)) { + GST_DEBUG_OBJECT (self, "found D3D11 memory feature"); + gst_caps_set_features (self->output_state->caps, 0, + gst_caps_features_new (GST_CAPS_FEATURE_MEMORY_D3D11_MEMORY, NULL)); + + self->use_d3d11_output = TRUE; + break; + } + } + } + gst_clear_caps (&peer_caps); + + return GST_VIDEO_DECODER_CLASS (parent_class)->negotiate (decoder); +} + +static gboolean +gst_d3d11_h264_dec_decide_allocation (GstVideoDecoder * decoder, + GstQuery * query) +{ + GstD3D11H264Dec *self = GST_D3D11_H264_DEC (decoder); + GstCaps *outcaps; + GstBufferPool *pool = NULL; + guint n, size, min, max; + GstVideoInfo vinfo = { 0, }; + GstStructure *config; + GstD3D11AllocationParams *d3d11_params; + + GST_DEBUG_OBJECT (self, "decide allocation"); + + gst_query_parse_allocation (query, &outcaps, NULL); + + if (!outcaps) { + GST_DEBUG_OBJECT (self, "No output caps"); + return FALSE; + } + + gst_video_info_from_caps (&vinfo, outcaps); + n = gst_query_get_n_allocation_pools (query); + if (n > 0) + gst_query_parse_nth_allocation_pool (query, 0, &pool, &size, &min, &max); + + /* create our own pool */ + if (pool && (self->use_d3d11_output && !GST_D3D11_BUFFER_POOL (pool))) { + gst_object_unref (pool); + pool = NULL; + } + + if (!pool) { + if (self->use_d3d11_output) + pool = gst_d3d11_buffer_pool_new (self->device); + else + pool = gst_video_buffer_pool_new (); + + min = max = 0; + size = (guint) vinfo.size; + } + + config = gst_buffer_pool_get_config (pool); + gst_buffer_pool_config_set_params (config, outcaps, size, min, max); + gst_buffer_pool_config_add_option (config, GST_BUFFER_POOL_OPTION_VIDEO_META); + + if (self->use_d3d11_output) { + d3d11_params = gst_buffer_pool_config_get_d3d11_allocation_params (config); + if (!d3d11_params) + d3d11_params = gst_d3d11_allocation_params_new (&vinfo, 0, + D3D11_USAGE_DEFAULT, 0); + + /* dxva2 decoder uses non-resource format + * (e.g., use NV12 instead of R8 + R8G8 */ + d3d11_params->desc[0].Width = GST_VIDEO_INFO_WIDTH (&vinfo); + d3d11_params->desc[0].Height = GST_VIDEO_INFO_HEIGHT (&vinfo); + d3d11_params->desc[0].Format = d3d11_params->d3d11_format->dxgi_format; + + d3d11_params->flags &= ~GST_D3D11_ALLOCATION_FLAG_USE_RESOURCE_FORMAT; + + gst_buffer_pool_config_set_d3d11_allocation_params (config, d3d11_params); + gst_d3d11_allocation_params_free (d3d11_params); + } + + gst_buffer_pool_set_config (pool, config); + if (self->use_d3d11_output) + size = GST_D3D11_BUFFER_POOL (pool)->buffer_size; + + if (n > 0) + gst_query_set_nth_allocation_pool (query, 0, pool, size, min, max); + else + gst_query_add_allocation_pool (query, pool, size, min, max); + gst_object_unref (pool); + + return GST_VIDEO_DECODER_CLASS (parent_class)->decide_allocation + (decoder, query); +} + +static gboolean +gst_d3d11_h264_dec_src_query (GstVideoDecoder * decoder, GstQuery * query) +{ + GstD3D11H264Dec *self = GST_D3D11_H264_DEC (decoder); + + switch (GST_QUERY_TYPE (query)) { + case GST_QUERY_CONTEXT: + if (gst_d3d11_handle_context_query (GST_ELEMENT (decoder), + query, self->device)) { + return TRUE; + } + break; + default: + break; + } + + return GST_VIDEO_DECODER_CLASS (parent_class)->src_query (decoder, query); +} + +static gboolean +gst_d3d11_h264_dec_new_sequence (GstH264Decoder * decoder, + const GstH264SPS * sps) +{ + GstD3D11H264Dec *self = GST_D3D11_H264_DEC (decoder); + gint crop_width, crop_height; + gboolean modified = FALSE; + static const GUID *supported_profiles[] = { + &GST_GUID_D3D11_DECODER_PROFILE_H264_IDCT_FGT, + &GST_GUID_D3D11_DECODER_PROFILE_H264_VLD_NOFGT, + &GST_GUID_D3D11_DECODER_PROFILE_H264_VLD_FGT + }; + + GST_LOG_OBJECT (self, "new sequence"); + + if (sps->frame_cropping_flag) { + crop_width = sps->crop_rect_width; + crop_height = sps->crop_rect_height; + } else { + crop_width = sps->width; + crop_height = sps->height; + } + + if (self->width != crop_width || self->height != crop_height || + self->coded_width != sps->width || self->coded_height != sps->height) { + GST_INFO_OBJECT (self, "resolution changed %dx%d", crop_width, crop_height); + self->width = crop_width; + self->height = crop_height; + self->coded_width = sps->width; + self->coded_height = sps->height; + modified = TRUE; + } + + if (self->bitdepth != sps->bit_depth_luma_minus8 + 8) { + GST_INFO_OBJECT (self, "bitdepth changed"); + self->bitdepth = sps->bit_depth_luma_minus8 + 8; + modified = TRUE; + } + + if (self->chroma_format_idc != sps->chroma_format_idc) { + GST_INFO_OBJECT (self, "chroma format changed"); + self->chroma_format_idc = sps->chroma_format_idc; + modified = TRUE; + } + + if (modified || !self->d3d11_decoder->opened) { + GstVideoInfo info; + + self->out_format = GST_VIDEO_FORMAT_UNKNOWN; + + if (self->bitdepth == 8) { + if (self->chroma_format_idc == 1) + self->out_format = GST_VIDEO_FORMAT_NV12; + else { + GST_FIXME_OBJECT (self, "Could not support 8bits non-4:2:0 format"); + } + } else if (self->bitdepth == 10) { + if (self->chroma_format_idc == 1) + self->out_format = GST_VIDEO_FORMAT_P010_10LE; + else { + GST_FIXME_OBJECT (self, "Could not support 10bits non-4:2:0 format"); + } + } + + if (self->out_format == GST_VIDEO_FORMAT_UNKNOWN) { + GST_ERROR_OBJECT (self, "Could not support bitdepth/chroma format"); + return FALSE; + } + + /* allocated internal pool with coded width/height */ + gst_video_info_set_format (&info, + self->out_format, self->coded_width, self->coded_height); + + gst_d3d11_decoder_reset (self->d3d11_decoder); + if (!gst_d3d11_decoder_open (self->d3d11_decoder, GST_D3D11_CODEC_H264, + &info, NUM_OUTPUT_VIEW, supported_profiles, + G_N_ELEMENTS (supported_profiles))) { + GST_ERROR_OBJECT (self, "Failed to create decoder"); + return FALSE; + } + + if (!gst_video_decoder_negotiate (GST_VIDEO_DECODER (self))) { + GST_ERROR_OBJECT (self, "Failed to negotiate with downstream"); + return FALSE; + } + } + + return TRUE; +} + +static gboolean +gst_d3d11_h264_dec_get_bitstream_buffer (GstD3D11H264Dec * self) +{ + GST_TRACE_OBJECT (self, "Getting bitstream buffer"); + if (!gst_d3d11_decoder_get_decoder_buffer (self->d3d11_decoder, + D3D11_VIDEO_DECODER_BUFFER_BITSTREAM, &self->bitstream_buffer_size, + (gpointer *) & self->bitstream_buffer_bytes)) { + GST_ERROR_OBJECT (self, "Faild to get bitstream buffer"); + return FALSE; + } + + GST_TRACE_OBJECT (self, "Got bitstream buffer %p with size %d", + self->bitstream_buffer_bytes, self->bitstream_buffer_size); + self->current_offset = 0; + + return TRUE; +} + +static GstD3D11DecoderOutputView * +gst_d3d11_h264_dec_get_output_view_from_picture (GstD3D11H264Dec * self, + GstH264Picture * picture) +{ + GstBuffer *view_buffer; + GstD3D11DecoderOutputView *view; + + view_buffer = (GstBuffer *) gst_h264_picture_get_user_data (picture); + if (!view_buffer) { + GST_DEBUG_OBJECT (self, "current picture does not have output view buffer"); + return NULL; + } + + view = gst_d3d11_decoder_get_output_view_from_buffer (self->d3d11_decoder, + view_buffer); + if (!view) { + GST_DEBUG_OBJECT (self, "current picture does not have output view handle"); + return NULL; + } + + return view; +} + +static gboolean +gst_d3d11_h264_dec_start_picture (GstH264Decoder * decoder, + GstH264Picture * picture, GstH264Slice * slice, GstH264Dpb * dpb) +{ + GstD3D11H264Dec *self = GST_D3D11_H264_DEC (decoder); + GstD3D11DecoderOutputView *view; + gint i; + GArray *dpb_array; + + view = gst_d3d11_h264_dec_get_output_view_from_picture (self, picture); + if (!view) { + GST_ERROR_OBJECT (self, "current picture does not have output view handle"); + return FALSE; + } + + GST_TRACE_OBJECT (self, "Begin frame"); + + if (!gst_d3d11_decoder_begin_frame (self->d3d11_decoder, view, 0, NULL)) { + GST_ERROR_OBJECT (self, "Failed to begin frame"); + return FALSE; + } + + for (i = 0; i < 16; i++) { + self->ref_frame_list[i].bPicEntry = 0xFF; + self->field_order_cnt_list[i][0] = 0; + self->field_order_cnt_list[i][1] = 0; + self->frame_num_list[i] = 0; + } + self->used_for_reference_flags = 0; + self->non_existing_frame_flags = 0; + + dpb_array = gst_h264_dpb_get_pictures_all (dpb); + + for (i = 0; i < dpb_array->len; i++) { + guint ref = 3; + GstH264Picture *other = g_array_index (dpb_array, GstH264Picture *, i); + GstD3D11DecoderOutputView *other_view; + gint id = 0xff; + + if (!other->ref) + continue; + + other_view = gst_d3d11_h264_dec_get_output_view_from_picture (self, other); + + if (other_view) + id = other_view->view_id; + + self->ref_frame_list[i].Index7Bits = id; + self->ref_frame_list[i].AssociatedFlag = other->long_term; + self->field_order_cnt_list[i][0] = other->top_field_order_cnt; + self->field_order_cnt_list[i][1] = other->bottom_field_order_cnt; + self->frame_num_list[i] = self->ref_frame_list[i].AssociatedFlag + ? other->long_term_pic_num : other->frame_num; + self->used_for_reference_flags |= ref << (2 * i); + self->non_existing_frame_flags |= (other->nonexisting) << i; + } + + g_array_unref (dpb_array); + g_array_set_size (self->slice_list, 0); + + return gst_d3d11_h264_dec_get_bitstream_buffer (self); +} + +static gboolean +gst_d3d11_h264_dec_new_picture (GstH264Decoder * decoder, + GstH264Picture * picture) +{ + GstD3D11H264Dec *self = GST_D3D11_H264_DEC (decoder); + GstBuffer *view_buffer; + GstD3D11Memory *mem; + + view_buffer = gst_d3d11_decoder_get_output_view_buffer (self->d3d11_decoder); + if (!view_buffer) { + GST_ERROR_OBJECT (self, "No available output view buffer"); + return FALSE; + } + + mem = (GstD3D11Memory *) gst_buffer_peek_memory (view_buffer, 0); + + GST_LOG_OBJECT (self, "New output view buffer %" GST_PTR_FORMAT " (index %d)", + view_buffer, mem->subresource_index); + + gst_h264_picture_set_user_data (picture, + view_buffer, (GDestroyNotify) gst_buffer_unref); + + GST_LOG_OBJECT (self, "New h264picture %p", picture); + + gst_h264_picture_replace (&self->current_picture, picture); + + return TRUE; +} + +static GstFlowReturn +gst_d3d11_h264_dec_output_picture (GstH264Decoder * decoder, + GstH264Picture * picture) +{ + GstD3D11H264Dec *self = GST_D3D11_H264_DEC (decoder); + GList *pending_frames, *iter; + GstVideoCodecFrame *frame = NULL; + GstBuffer *output_buffer = NULL; + GstFlowReturn ret; + GstBuffer *view_buffer; + + GST_LOG_OBJECT (self, + "Outputting picture %p (poc %d)", picture, picture->pic_order_cnt); + + view_buffer = (GstBuffer *) gst_h264_picture_get_user_data (picture); + + if (!view_buffer) { + GST_ERROR_OBJECT (self, "Could not get output view"); + return FALSE; + } + + pending_frames = gst_video_decoder_get_frames (GST_VIDEO_DECODER (self)); + for (iter = pending_frames; iter; iter = g_list_next (iter)) { + GstVideoCodecFrame *tmp; + GstH264Picture *other_pic; + + tmp = (GstVideoCodecFrame *) iter->data; + other_pic = gst_video_codec_frame_get_user_data (tmp); + if (!other_pic) { + /* FIXME: what should we do here? */ + GST_WARNING_OBJECT (self, + "Codec frame %p does not have corresponding picture object", tmp); + continue; + } + + if (other_pic == picture) { + frame = gst_video_codec_frame_ref (tmp); + break; + } + } + + g_list_free_full (pending_frames, + (GDestroyNotify) gst_video_codec_frame_unref); + + if (!frame) { + GST_WARNING_OBJECT (self, + "Failed to find codec frame for picture %p", picture); + + output_buffer = + gst_video_decoder_allocate_output_buffer (GST_VIDEO_DECODER (self)); + + if (!output_buffer) { + GST_ERROR_OBJECT (self, "Couldn't allocate output buffer"); + return GST_FLOW_ERROR; + } + + GST_BUFFER_PTS (output_buffer) = picture->pts; + GST_BUFFER_DTS (output_buffer) = GST_CLOCK_TIME_NONE; + GST_BUFFER_DURATION (output_buffer) = GST_CLOCK_TIME_NONE; + } else { + ret = + gst_video_decoder_allocate_output_frame (GST_VIDEO_DECODER (self), + frame); + + if (ret != GST_FLOW_OK) { + GST_ERROR_OBJECT (self, "failed to allocate output frame"); + return ret; + } + + output_buffer = frame->output_buffer; + GST_BUFFER_PTS (output_buffer) = GST_BUFFER_PTS (frame->input_buffer); + GST_BUFFER_DTS (output_buffer) = GST_CLOCK_TIME_NONE; + GST_BUFFER_DURATION (output_buffer) = + GST_BUFFER_DURATION (frame->input_buffer); + } + + if (!gst_d3d11_decoder_copy_decoder_buffer (self->d3d11_decoder, + &self->output_state->info, view_buffer, output_buffer)) { + GST_ERROR_OBJECT (self, "Failed to copy buffer"); + if (frame) + gst_video_decoder_drop_frame (GST_VIDEO_DECODER (self), frame); + else + gst_buffer_unref (output_buffer); + + return GST_FLOW_ERROR; + } + + GST_LOG_OBJECT (self, "Finish frame %" GST_TIME_FORMAT, + GST_TIME_ARGS (GST_BUFFER_PTS (output_buffer))); + + if (frame) { + ret = gst_video_decoder_finish_frame (GST_VIDEO_DECODER (self), frame); + } else { + ret = gst_pad_push (GST_VIDEO_DECODER_SRC_PAD (self), output_buffer); + } + + return ret; +} + +static gboolean +gst_d3d11_h264_dec_submit_slice_data (GstD3D11H264Dec * self) +{ + guint buffer_size; + gpointer buffer; + guint8 *data; + gsize offset = 0; + gint i; + D3D11_VIDEO_DECODER_BUFFER_DESC buffer_desc[4] = { 0, }; + gboolean ret; + + if (self->slice_list->len < 1) { + GST_WARNING_OBJECT (self, "Nothing to submit"); + return FALSE; + } + + GST_TRACE_OBJECT (self, "Getting slice control buffer"); + + if (!gst_d3d11_decoder_get_decoder_buffer (self->d3d11_decoder, + D3D11_VIDEO_DECODER_BUFFER_SLICE_CONTROL, &buffer_size, &buffer)) { + GST_ERROR_OBJECT (self, "Couldn't get slice control buffer"); + return FALSE; + } + + data = buffer; + for (i = 0; i < self->slice_list->len; i++) { + DXVA_Slice_H264_Short *slice_data = + &g_array_index (self->slice_list, DXVA_Slice_H264_Short, i); + + memcpy (data + offset, slice_data, sizeof (DXVA_Slice_H264_Short)); + offset += sizeof (DXVA_Slice_H264_Short); + } + + GST_TRACE_OBJECT (self, "Release slice control buffer"); + if (!gst_d3d11_decoder_release_decoder_buffer (self->d3d11_decoder, + D3D11_VIDEO_DECODER_BUFFER_SLICE_CONTROL)) { + GST_ERROR_OBJECT (self, "Failed to release slice control buffer"); + return FALSE; + } + + if (!gst_d3d11_decoder_release_decoder_buffer (self->d3d11_decoder, + D3D11_VIDEO_DECODER_BUFFER_BITSTREAM)) { + GST_ERROR_OBJECT (self, "Failed to release bitstream buffer"); + return FALSE; + } + + buffer_desc[0].BufferType = D3D11_VIDEO_DECODER_BUFFER_PICTURE_PARAMETERS; + buffer_desc[0].DataOffset = 0; + buffer_desc[0].DataSize = sizeof (DXVA_PicParams_H264); + + buffer_desc[1].BufferType = + D3D11_VIDEO_DECODER_BUFFER_INVERSE_QUANTIZATION_MATRIX; + buffer_desc[1].DataOffset = 0; + buffer_desc[1].DataSize = sizeof (DXVA_Qmatrix_H264); + + buffer_desc[2].BufferType = D3D11_VIDEO_DECODER_BUFFER_SLICE_CONTROL; + buffer_desc[2].DataOffset = 0; + buffer_desc[2].DataSize = + sizeof (DXVA_Slice_H264_Short) * self->slice_list->len; + + buffer_desc[3].BufferType = D3D11_VIDEO_DECODER_BUFFER_BITSTREAM; + buffer_desc[3].DataOffset = 0; + buffer_desc[3].DataSize = self->current_offset; + + ret = gst_d3d11_decoder_submit_decoder_buffers (self->d3d11_decoder, + 4, buffer_desc); + + self->current_offset = 0; + self->bitstream_buffer_bytes = NULL; + self->bitstream_buffer_size = 0; + g_array_set_size (self->slice_list, 0); + + return ret; +} + +static gboolean +gst_d3d11_h264_dec_end_picture (GstH264Decoder * decoder, + GstH264Picture * picture) +{ + GstD3D11H264Dec *self = GST_D3D11_H264_DEC (decoder); + + GST_LOG_OBJECT (self, "end picture %p, (poc %d)", + picture, picture->pic_order_cnt); + + if (!gst_d3d11_h264_dec_submit_slice_data (self)) { + GST_ERROR_OBJECT (self, "Failed to submit slice data"); + return FALSE; + } + + if (!gst_d3d11_decoder_end_frame (self->d3d11_decoder)) { + GST_ERROR_OBJECT (self, "Failed to EndFrame"); + return FALSE; + } + + return TRUE; +} + +static void +gst_d3d11_h264_dec_picture_params_from_sps (GstD3D11H264Dec * self, + const GstH264SPS * sps, gboolean field_pic, DXVA_PicParams_H264 * params) +{ +#define COPY_FIELD(f) \ + (params)->f = (sps)->f + + params->wFrameWidthInMbsMinus1 = sps->pic_width_in_mbs_minus1; + params->wFrameHeightInMbsMinus1 = sps->pic_height_in_map_units_minus1; + params->residual_colour_transform_flag = sps->separate_colour_plane_flag; + params->MbaffFrameFlag = sps->mb_adaptive_frame_field_flag && field_pic; + params->field_pic_flag = field_pic; + params->MinLumaBipredSize8x8Flag = sps->level_idc >= 31; + + COPY_FIELD (num_ref_frames); + COPY_FIELD (chroma_format_idc); + COPY_FIELD (frame_mbs_only_flag); + COPY_FIELD (bit_depth_luma_minus8); + COPY_FIELD (bit_depth_chroma_minus8); + COPY_FIELD (log2_max_frame_num_minus4); + COPY_FIELD (pic_order_cnt_type); + COPY_FIELD (log2_max_pic_order_cnt_lsb_minus4); + COPY_FIELD (delta_pic_order_always_zero_flag); + COPY_FIELD (direct_8x8_inference_flag); + +#undef COPY_FIELD +} + +static void +gst_d3d11_h264_dec_picture_params_from_pps (GstD3D11H264Dec * self, + const GstH264PPS * pps, DXVA_PicParams_H264 * params) +{ +#define COPY_FIELD(f) \ + (params)->f = (pps)->f + + COPY_FIELD (constrained_intra_pred_flag); + COPY_FIELD (weighted_pred_flag); + COPY_FIELD (weighted_bipred_idc); + COPY_FIELD (transform_8x8_mode_flag); + COPY_FIELD (pic_init_qs_minus26); + COPY_FIELD (chroma_qp_index_offset); + COPY_FIELD (second_chroma_qp_index_offset); + COPY_FIELD (pic_init_qp_minus26); + COPY_FIELD (num_ref_idx_l0_active_minus1); + COPY_FIELD (num_ref_idx_l1_active_minus1); + COPY_FIELD (entropy_coding_mode_flag); + COPY_FIELD (pic_order_present_flag); + COPY_FIELD (deblocking_filter_control_present_flag); + COPY_FIELD (redundant_pic_cnt_present_flag); + COPY_FIELD (num_slice_groups_minus1); + COPY_FIELD (slice_group_map_type); + +#undef COPY_FIELD +} + +static void +gst_d3d11_h264_dec_picture_params_from_slice_header (GstD3D11H264Dec * + self, const GstH264SliceHdr * slice_header, DXVA_PicParams_H264 * params) +{ + params->sp_for_switch_flag = slice_header->sp_for_switch_flag; + params->field_pic_flag = slice_header->field_pic_flag; + params->CurrPic.AssociatedFlag = slice_header->bottom_field_flag; + params->IntraPicFlag = + GST_H264_IS_I_SLICE (slice_header) || GST_H264_IS_SI_SLICE (slice_header); +} + +static gboolean +gst_d3d11_h264_dec_fill_picture_params (GstD3D11H264Dec * self, + const GstH264SliceHdr * slice_header, DXVA_PicParams_H264 * params) +{ + const GstH264SPS *sps; + const GstH264PPS *pps; + + g_return_val_if_fail (slice_header->pps != NULL, FALSE); + g_return_val_if_fail (slice_header->pps->sequence != NULL, FALSE); + + pps = slice_header->pps; + sps = pps->sequence; + + memset (params, 0, sizeof (DXVA_PicParams_H264)); + + params->MbsConsecutiveFlag = 1; + params->Reserved16Bits = 3; + params->ContinuationFlag = 1; + params->Reserved8BitsA = 0; + params->Reserved8BitsB = 0; + params->StatusReportFeedbackNumber = 1; + + gst_d3d11_h264_dec_picture_params_from_sps (self, + sps, slice_header->field_pic_flag, params); + gst_d3d11_h264_dec_picture_params_from_pps (self, pps, params); + gst_d3d11_h264_dec_picture_params_from_slice_header (self, + slice_header, params); + + return TRUE; +} + +static gboolean +gst_d3d11_h264_dec_decode_slice (GstH264Decoder * decoder, + GstH264Picture * picture, GstH264Slice * slice) +{ + GstD3D11H264Dec *self = GST_D3D11_H264_DEC (decoder); + GstH264SPS *sps; + GstH264PPS *pps; + DXVA_PicParams_H264 pic_params = { 0, }; + DXVA_Qmatrix_H264 iq_matrix = { 0, }; + guint d3d11_buffer_size = 0; + gpointer d3d11_buffer = NULL; + gint i, j; + GstD3D11DecoderOutputView *view; + + pps = slice->header.pps; + sps = pps->sequence; + + view = gst_d3d11_h264_dec_get_output_view_from_picture (self, picture); + + if (!view) { + GST_ERROR_OBJECT (self, "current picture does not have output view"); + return FALSE; + } + + gst_d3d11_h264_dec_fill_picture_params (self, &slice->header, &pic_params); + + pic_params.CurrPic.Index7Bits = view->view_id; + pic_params.RefPicFlag = picture->ref; + pic_params.frame_num = picture->frame_num; + + if (pic_params.field_pic_flag && pic_params.CurrPic.AssociatedFlag) { + pic_params.CurrFieldOrderCnt[1] = picture->bottom_field_order_cnt; + pic_params.CurrFieldOrderCnt[0] = 0; + } else if (pic_params.field_pic_flag && !pic_params.CurrPic.AssociatedFlag) { + pic_params.CurrFieldOrderCnt[0] = picture->top_field_order_cnt; + pic_params.CurrFieldOrderCnt[1] = 0; + } else { + pic_params.CurrFieldOrderCnt[0] = picture->top_field_order_cnt; + pic_params.CurrFieldOrderCnt[1] = picture->bottom_field_order_cnt; + } + + memcpy (pic_params.RefFrameList, self->ref_frame_list, + sizeof (pic_params.RefFrameList)); + memcpy (pic_params.FieldOrderCntList, self->field_order_cnt_list, + sizeof (pic_params.FieldOrderCntList)); + memcpy (pic_params.FrameNumList, self->frame_num_list, + sizeof (pic_params.FrameNumList)); + + pic_params.UsedForReferenceFlags = self->used_for_reference_flags; + pic_params.NonExistingFrameFlags = self->non_existing_frame_flags; + + GST_TRACE_OBJECT (self, "Getting picture param decoder buffer"); + + if (!gst_d3d11_decoder_get_decoder_buffer (self->d3d11_decoder, + D3D11_VIDEO_DECODER_BUFFER_PICTURE_PARAMETERS, &d3d11_buffer_size, + &d3d11_buffer)) { + GST_ERROR_OBJECT (self, + "Failed to get decoder buffer for picture parameters"); + return FALSE; + } + + memcpy (d3d11_buffer, &pic_params, sizeof (DXVA_PicParams_H264)); + + GST_TRACE_OBJECT (self, "Release picture param decoder buffer"); + + if (!gst_d3d11_decoder_release_decoder_buffer (self->d3d11_decoder, + D3D11_VIDEO_DECODER_BUFFER_PICTURE_PARAMETERS)) { + GST_ERROR_OBJECT (self, "Failed to release decoder buffer"); + return FALSE; + } + + if (pps->pic_scaling_matrix_present_flag) { + for (i = 0; i < 6; i++) { + for (j = 0; j < 16; j++) { + iq_matrix.bScalingLists4x4[i][j] = pps->scaling_lists_4x4[i][j]; + } + } + + for (i = 0; i < 2; i++) { + for (j = 0; j < 64; j++) { + iq_matrix.bScalingLists8x8[i][j] = pps->scaling_lists_8x8[i][j]; + } + } + } else { + for (i = 0; i < 6; i++) { + for (j = 0; j < 16; j++) { + iq_matrix.bScalingLists4x4[i][j] = sps->scaling_lists_4x4[i][j]; + } + } + + for (i = 0; i < 2; i++) { + for (j = 0; j < 64; j++) { + iq_matrix.bScalingLists8x8[i][j] = sps->scaling_lists_8x8[i][j]; + } + } + } + + GST_TRACE_OBJECT (self, "Getting inverse quantization maxtirx buffer"); + + if (!gst_d3d11_decoder_get_decoder_buffer (self->d3d11_decoder, + D3D11_VIDEO_DECODER_BUFFER_INVERSE_QUANTIZATION_MATRIX, + &d3d11_buffer_size, &d3d11_buffer)) { + GST_ERROR_OBJECT (self, + "Failed to get decoder buffer for inv. quantization matrix"); + return FALSE; + } + + memcpy (d3d11_buffer, &iq_matrix, sizeof (DXVA_Qmatrix_H264)); + + GST_TRACE_OBJECT (self, "Release inverse quantization maxtirx buffer"); + + if (!gst_d3d11_decoder_release_decoder_buffer (self->d3d11_decoder, + D3D11_VIDEO_DECODER_BUFFER_INVERSE_QUANTIZATION_MATRIX)) { + GST_ERROR_OBJECT (self, "Failed to release decoder buffer"); + return FALSE; + } + + { + guint to_write = slice->nalu.size + 3; + gboolean is_first = TRUE; + + while (to_write > 0) { + guint bytes_to_copy; + gboolean is_last = TRUE; + DXVA_Slice_H264_Short slice_short = { 0, }; + + if (self->bitstream_buffer_size < to_write && self->slice_list->len > 0) { + if (!gst_d3d11_h264_dec_submit_slice_data (self)) { + GST_ERROR_OBJECT (self, "Failed to submit bitstream buffers"); + return FALSE; + } + + if (!gst_d3d11_h264_dec_get_bitstream_buffer (self)) { + GST_ERROR_OBJECT (self, "Failed to get bitstream buffer"); + return FALSE; + } + } + + bytes_to_copy = to_write; + + if (bytes_to_copy > self->bitstream_buffer_size) { + bytes_to_copy = self->bitstream_buffer_size; + is_last = FALSE; + } + + if (bytes_to_copy >= 3 && is_first) { + /* normal case */ + self->bitstream_buffer_bytes[0] = 0; + self->bitstream_buffer_bytes[1] = 0; + self->bitstream_buffer_bytes[2] = 1; + memcpy (self->bitstream_buffer_bytes + 3, + slice->nalu.data + slice->nalu.offset, bytes_to_copy - 3); + } else { + /* when this nal unit date is splitted into two buffer */ + memcpy (self->bitstream_buffer_bytes, + slice->nalu.data + slice->nalu.offset, bytes_to_copy); + } + + slice_short.BSNALunitDataLocation = self->current_offset; + slice_short.SliceBytesInBuffer = bytes_to_copy; + /* wBadSliceChopping: (dxva h264 spec.) + * 0: All bits for the slice are located within the corresponding + * bitstream data buffer + * 1: The bitstream data buffer contains the start of the slice, + * but not the entire slice, because the buffer is full + * 2: The bitstream data buffer contains the end of the slice. + * It does not contain the start of the slice, because the start of + * the slice was located in the previous bitstream data buffer. + * 3: The bitstream data buffer does not contain the start of the slice + * (because the start of the slice was located in the previous + * bitstream data buffer), and it does not contain the end of the slice + * (because the current bitstream data buffer is also full). + */ + if (is_last && is_first) { + slice_short.wBadSliceChopping = 0; + } else if (!is_last && is_first) { + slice_short.wBadSliceChopping = 1; + } else if (is_last && !is_first) { + slice_short.wBadSliceChopping = 2; + } else { + slice_short.wBadSliceChopping = 3; + } + + g_array_append_val (self->slice_list, slice_short); + self->bitstream_buffer_size -= bytes_to_copy; + self->current_offset += bytes_to_copy; + self->bitstream_buffer_bytes += bytes_to_copy; + is_first = FALSE; + to_write -= bytes_to_copy; + } + } + + return TRUE; +} diff --git a/sys/d3d11/gstd3d11h264dec.h b/sys/d3d11/gstd3d11h264dec.h new file mode 100644 index 0000000000..890e736dbb --- /dev/null +++ b/sys/d3d11/gstd3d11h264dec.h @@ -0,0 +1,87 @@ +/* GStreamer + * Copyright (C) 2019 Seungha Yang + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifndef __GST_D3D11_H264_DEC_H__ +#define __GST_D3D11_H264_DEC_H__ + +#include "gsth264decoder.h" +#include "gsth264picture.h" +#include "gstd3d11decoder.h" + +G_BEGIN_DECLS + +#define GST_TYPE_D3D11_H264_DEC \ + (gst_d3d11_h264_dec_get_type()) +#define GST_D3D11_H264_DEC(obj) \ + (G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_D3D11_H264_DEC,GstD3D11H264Dec)) +#define GST_D3D11_H264_DEC_CLASS(klass) \ + (G_TYPE_CHECK_CLASS_CAST((klass),GST_TYPE_D3D11_H264_DEC,GstD3D11H264DecClass)) +#define GST_D3D11_H264_DEC_GET_CLASS(obj) \ + (G_TYPE_INSTANCE_GET_CLASS((obj),GST_TYPE_D3D11_H264_DEC,GstD3D11H264DecClass)) +#define GST_IS_D3D11_H264_DEC(obj) \ + (G_TYPE_CHECK_INSTANCE_TYPE((obj),GST_TYPE_D3D11_H264_DEC)) +#define GST_IS_D3D11_H264_DEC_CLASS(klass) \ + (G_TYPE_CHECK_CLASS_TYPE((klass),GST_TYPE_D3D11_H264_DEC)) + +struct _GstD3D11H264Dec +{ + GstH264Decoder parent; + + GstVideoCodecState *output_state; + + GstD3D11Device *device; + gint adapter; + + guint width, height; + guint coded_width, coded_height; + guint bitdepth; + guint chroma_format_idc; + GstVideoFormat out_format; + + DXVA_PicEntry_H264 ref_frame_list[16]; + INT field_order_cnt_list[16][2]; + USHORT frame_num_list[16]; + UINT used_for_reference_flags; + USHORT non_existing_frame_flags; + + /* Array of DXVA_Slice_H264_Short */ + GArray *slice_list; + + GstD3D11Decoder *d3d11_decoder; + + GstH264Picture *current_picture; + + /* Pointing current bitstream buffer */ + guint current_offset; + guint bitstream_buffer_size; + guint8 * bitstream_buffer_bytes; + + gboolean use_d3d11_output; +}; + +struct _GstD3D11H264DecClass +{ + GstH264DecoderClass parent_class; +}; + +GType gst_d3d11_h264_dec_get_type (void); + +G_END_DECLS + +#endif /* __GST_D3D11_H264_DEC_H__ */ diff --git a/sys/d3d11/gsth264decoder.c b/sys/d3d11/gsth264decoder.c new file mode 100644 index 0000000000..d8a1b16bc2 --- /dev/null +++ b/sys/d3d11/gsth264decoder.c @@ -0,0 +1,1829 @@ +/* GStreamer + * Copyright (C) 2019 Seungha Yang + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + * + * NOTE: some of implementations are copied/modified from Chromium code + * + * Copyright 2015 The Chromium Authors. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include "gsth264decoder.h" + +GST_DEBUG_CATEGORY_EXTERN (gst_d3d11_h264_dec_debug); +#define GST_CAT_DEFAULT gst_d3d11_h264_dec_debug + +typedef enum +{ + GST_H264_DECODER_FORMAT_NONE, + GST_H264_DECODER_FORMAT_AVC, + GST_H264_DECODER_FORMAT_BYTE +} GstH264DecoderFormat; + +typedef enum +{ + GST_H264_DECODER_ALIGN_NONE, + GST_H264_DECODER_ALIGN_NAL, + GST_H264_DECODER_ALIGN_AU +} GstH264DecoderAlign; + +struct _GstH264DecoderPrivate +{ + gint width, height; + gint fps_num, fps_den; + gint upstream_par_n, upstream_par_d; + gint parsed_par_n, parsed_par_d; + gint parsed_fps_n, parsed_fps_d; + GstVideoColorimetry parsed_colorimetry; + /* input codec_data, if any */ + GstBuffer *codec_data; + guint nal_length_size; + + /* state */ + GstH264DecoderFormat in_format; + GstH264DecoderAlign align; + GstH264NalParser *parser; + GstH264Dpb *dpb; + GstFlowReturn last_ret; + + /* sps/pps of the current slice */ + const GstH264SPS *active_sps; + const GstH264PPS *active_pps; + + /* Picture currently being processed/decoded */ + GstH264Picture *current_picture; + + /* Slice (slice header + nalu) currently being processed/decodec */ + GstH264Slice current_slice; + + gint max_frame_num; + gint max_pic_num; + gint max_long_term_frame_idx; + gsize max_num_reorder_frames; + + gint prev_frame_num; + gint prev_ref_frame_num; + gint prev_frame_num_offset; + gboolean prev_has_memmgmnt5; + + /* Values related to previously decoded reference picture */ + gboolean prev_ref_has_memmgmnt5; + gint prev_ref_top_field_order_cnt; + gint prev_ref_pic_order_cnt_msb; + gint prev_ref_pic_order_cnt_lsb; + + GstH264PictureField prev_ref_field; + + /* PicOrderCount of the previously outputted frame */ + gint last_output_poc; +}; + +#define parent_class gst_h264_decoder_parent_class +G_DEFINE_ABSTRACT_TYPE_WITH_PRIVATE (GstH264Decoder, gst_h264_decoder, + GST_TYPE_VIDEO_DECODER); + +static gboolean gst_h264_decoder_start (GstVideoDecoder * decoder); +static gboolean gst_h264_decoder_stop (GstVideoDecoder * decoder); +static GstFlowReturn gst_h264_decoder_parse (GstVideoDecoder * decoder, + GstVideoCodecFrame * frame, GstAdapter * adapter, gboolean at_eos); +static gboolean gst_h264_decoder_set_format (GstVideoDecoder * decoder, + GstVideoCodecState * state); +static GstFlowReturn gst_h264_decoder_finish (GstVideoDecoder * decoder); +static gboolean gst_h264_decoder_flush (GstVideoDecoder * decoder); +static GstFlowReturn gst_h264_decoder_drain (GstVideoDecoder * decoder); + +/* codec spcific functions */ +static gboolean gst_h264_decoder_process_sps (GstH264Decoder * self, + GstH264SPS * sps); +static gboolean gst_h264_decoder_decode_slice (GstH264Decoder * self); +static gboolean gst_h264_decoder_fill_picture_from_slice (GstH264Decoder * self, + const GstH264Slice * slice, GstH264Picture * picture); +static gboolean gst_h264_decoder_calculate_poc (GstH264Decoder * self, + GstH264Picture * picture); +static gboolean gst_h264_decoder_init_gap_picture (GstH264Decoder * self, + GstH264Picture * picture, gint frame_num); +static gboolean +gst_h264_decoder_output_all_remaining_pics (GstH264Decoder * self); +static gboolean gst_h264_decoder_finish_current_picture (GstH264Decoder * self); +static gboolean gst_h264_decoder_finish_picture (GstH264Decoder * self, + GstH264Picture * picture); + +static void +gst_h264_decoder_class_init (GstH264DecoderClass * klass) +{ + GstVideoDecoderClass *decoder_class = GST_VIDEO_DECODER_CLASS (klass); + + decoder_class->start = GST_DEBUG_FUNCPTR (gst_h264_decoder_start); + decoder_class->stop = GST_DEBUG_FUNCPTR (gst_h264_decoder_stop); + decoder_class->parse = GST_DEBUG_FUNCPTR (gst_h264_decoder_parse); + decoder_class->set_format = GST_DEBUG_FUNCPTR (gst_h264_decoder_set_format); + decoder_class->finish = GST_DEBUG_FUNCPTR (gst_h264_decoder_finish); + decoder_class->flush = GST_DEBUG_FUNCPTR (gst_h264_decoder_flush); + decoder_class->drain = GST_DEBUG_FUNCPTR (gst_h264_decoder_drain); +} + +static void +gst_h264_decoder_init (GstH264Decoder * self) +{ + gst_video_decoder_set_packetized (GST_VIDEO_DECODER (self), FALSE); + + self->priv = gst_h264_decoder_get_instance_private (self); +} + +static gboolean +gst_h264_decoder_start (GstVideoDecoder * decoder) +{ + GstH264Decoder *self = GST_H264_DECODER (decoder); + GstH264DecoderPrivate *priv = self->priv; + + priv->parser = gst_h264_nal_parser_new (); + priv->dpb = gst_h264_dpb_new (); + + return TRUE; +} + +static gboolean +gst_h264_decoder_stop (GstVideoDecoder * decoder) +{ + GstH264Decoder *self = GST_H264_DECODER (decoder); + GstH264DecoderPrivate *priv = self->priv; + + if (self->input_state) { + gst_video_codec_state_unref (self->input_state); + self->input_state = NULL; + } + + gst_clear_buffer (&priv->codec_data); + + if (priv->parser) { + gst_h264_nal_parser_free (priv->parser); + priv->parser = NULL; + } + + if (priv->dpb) { + gst_h264_dpb_free (priv->dpb); + priv->dpb = NULL; + } + + return TRUE; +} + +static void +gst_h264_decoder_clear_dpb (GstH264Decoder * self) +{ + GstH264DecoderPrivate *priv = self->priv; + + gst_h264_dpb_clear (priv->dpb); + priv->last_output_poc = -1; +} + +static gboolean +gst_h264_decoder_flush (GstVideoDecoder * decoder) +{ + GstH264Decoder *self = GST_H264_DECODER (decoder); + gboolean ret = TRUE; + + gst_h264_decoder_finish_current_picture (self); + + if (!gst_h264_decoder_output_all_remaining_pics (self)) + ret = FALSE; + + gst_h264_decoder_clear_dpb (self); + + return ret; +} + +static GstFlowReturn +gst_h264_decoder_drain (GstVideoDecoder * decoder) +{ + GstH264Decoder *self = GST_H264_DECODER (decoder); + GstH264DecoderPrivate *priv = self->priv; + + gst_h264_decoder_flush (decoder); + + return priv->last_ret; +} + +static GstFlowReturn +gst_h264_decoder_finish (GstVideoDecoder * decoder) +{ + return gst_h264_decoder_drain (decoder); +} + +static gboolean +gst_h264_decoder_parse_sps (GstH264Decoder * self, GstH264NalUnit * nalu) +{ + GstH264DecoderPrivate *priv = self->priv; + GstH264SPS sps; + GstH264ParserResult pres; + gboolean ret = TRUE; + + gst_h264_decoder_finish_current_picture (self); + + pres = gst_h264_parser_parse_sps (priv->parser, nalu, &sps); + if (pres != GST_H264_PARSER_OK) { + GST_WARNING_OBJECT (self, "Failed to parse SPS, result %d", pres); + return FALSE; + } + + GST_LOG_OBJECT (self, "SPS parsed"); + + if (!gst_h264_decoder_process_sps (self, &sps)) + ret = FALSE; + + gst_h264_sps_clear (&sps); + + return ret; +} + +static gboolean +gst_h264_decoder_parse_pps (GstH264Decoder * self, GstH264NalUnit * nalu) +{ + GstH264DecoderPrivate *priv = self->priv; + GstH264PPS pps; + GstH264ParserResult pres; + + gst_h264_decoder_finish_current_picture (self); + + pres = gst_h264_parser_parse_pps (priv->parser, nalu, &pps); + if (pres != GST_H264_PARSER_OK) { + GST_WARNING_OBJECT (self, "Failed to parse PPS, result %d", pres); + return FALSE; + } + + GST_LOG_OBJECT (self, "PPS parsed"); + gst_h264_pps_clear (&pps); + + return TRUE; +} + +static gboolean +gst_h264_decoder_parse_codec_data (GstH264Decoder * self, const guint8 * data, + gsize size) +{ + GstH264DecoderPrivate *priv = self->priv; + guint num_sps, num_pps; + guint off; + gint i; + GstH264ParserResult pres; + GstH264NalUnit nalu; +#ifndef GST_DISABLE_GST_DEBUG + guint profile; +#endif + + /* parse the avcC data */ + if (size < 7) { /* when numSPS==0 and numPPS==0, length is 7 bytes */ + return FALSE; + } + + /* parse the version, this must be 1 */ + if (data[0] != 1) { + return FALSE; + } +#ifndef GST_DISABLE_GST_DEBUG + /* AVCProfileIndication */ + /* profile_compat */ + /* AVCLevelIndication */ + profile = (data[1] << 16) | (data[2] << 8) | data[3]; + GST_DEBUG_OBJECT (self, "profile %06x", profile); +#endif + + /* 6 bits reserved | 2 bits lengthSizeMinusOne */ + /* this is the number of bytes in front of the NAL units to mark their + * length */ + priv->nal_length_size = (data[4] & 0x03) + 1; + GST_DEBUG_OBJECT (self, "nal length size %u", priv->nal_length_size); + + num_sps = data[5] & 0x1f; + off = 6; + for (i = 0; i < num_sps; i++) { + pres = gst_h264_parser_identify_nalu_avc (priv->parser, + data, off, size, 2, &nalu); + if (pres != GST_H264_PARSER_OK) { + GST_WARNING_OBJECT (self, "Failed to identify SPS nalu"); + return FALSE; + } + + gst_h264_decoder_parse_sps (self, &nalu); + off = nalu.offset + nalu.size; + } + + if (off >= size) { + GST_WARNING_OBJECT (self, "Too small avcC"); + return FALSE; + } + + num_pps = data[off]; + off++; + + for (i = 0; i < num_pps; i++) { + pres = gst_h264_parser_identify_nalu_avc (priv->parser, + data, off, size, 2, &nalu); + if (pres != GST_H264_PARSER_OK) { + GST_WARNING_OBJECT (self, "Failed to identify PPS nalu"); + return FALSE; + } + + gst_h264_decoder_parse_pps (self, &nalu); + off = nalu.offset + nalu.size; + } + + return TRUE; +} + +static gboolean +gst_h264_decoder_is_new_au (GstH264Decoder * self, GstH264Slice * slice) +{ + GstH264DecoderPrivate *priv = self->priv; + GstH264SliceHdr *slice_hdr; + GstH264NalUnit *nalu; + GstH264Picture *current_picture; + const GstH264SPS *sps; + + current_picture = priv->current_picture; + if (!current_picture) + return TRUE; + + slice_hdr = &slice->header; + nalu = &slice->nalu; + if (slice_hdr->frame_num != current_picture->frame_num || + slice_hdr->pps->id != priv->active_pps->id || + nalu->ref_idc != current_picture->nal_ref_idc || + (! !nalu->idr_pic_flag) != (! !current_picture->idr_pic_id) || + (nalu->idr_pic_flag && + (slice_hdr->idr_pic_id != current_picture->idr_pic_id || + !slice_hdr->first_mb_in_slice))) + return TRUE; + + sps = priv->active_sps; + if (!sps) + return FALSE; + + if (sps->pic_order_cnt_type == current_picture->pic_order_cnt_type) { + if (current_picture->pic_order_cnt_type == 0) { + if (slice_hdr->pic_order_cnt_lsb != current_picture->pic_order_cnt_lsb || + slice_hdr->delta_pic_order_cnt_bottom != + current_picture->delta_pic_order_cnt_bottom) + return TRUE; + } else if (current_picture->pic_order_cnt_type == 1) { + if (slice_hdr->delta_pic_order_cnt[0] != + current_picture->delta_pic_order_cnt0 + || slice_hdr->delta_pic_order_cnt[1] != + current_picture->delta_pic_order_cnt1) + return TRUE; + } + } + + return FALSE; +} + +static gboolean +gst_h264_decoder_preprocess_slice (GstH264Decoder * self, GstH264Slice * slice) +{ + GstH264DecoderPrivate *priv = self->priv; + + if (gst_h264_decoder_is_new_au (self, slice)) { + /* finish previous frame if any */ + if (!gst_h264_decoder_finish_current_picture (self)) + return FALSE; + + if (slice->header.first_mb_in_slice != 0) { + GST_ERROR_OBJECT (self, "Invalid stream, first_mb_in_slice %d", + slice->header.first_mb_in_slice); + return FALSE; + } + + /* If the new picture is an IDR, flush DPB */ + if (slice->nalu.idr_pic_flag) { + /* Output all remaining pictures, unless we are explicitly instructed + * not to do so */ + if (!slice->header.dec_ref_pic_marking.no_output_of_prior_pics_flag) + gst_h264_decoder_flush (GST_VIDEO_DECODER (self)); + + gst_h264_dpb_clear (priv->dpb); + } + } + + return TRUE; +} + +static void +gst_h264_decoder_update_pic_nums (GstH264Decoder * self, gint frame_num) +{ + GstH264DecoderPrivate *priv = self->priv; + GArray *dpb = gst_h264_dpb_get_pictures_all (priv->dpb); + gint i; + + for (i = 0; i < dpb->len; i++) { + GstH264Picture *picture = g_array_index (dpb, GstH264Picture *, i); + + if (picture->field != GST_H264_PICTURE_FIELD_FRAME) { + GST_FIXME_OBJECT (self, "Interlaced video not supported"); + continue; + } + + if (!picture->ref) + continue; + + if (picture->long_term) { + picture->long_term_pic_num = picture->long_term_frame_idx; + } else { + if (picture->frame_num > frame_num) + picture->frame_num_wrap = picture->frame_num - priv->max_frame_num; + else + picture->frame_num_wrap = picture->frame_num; + + picture->pic_num = picture->frame_num_wrap; + } + } + + g_array_unref (dpb); +} + +static gboolean +gst_h264_decoder_handle_frame_num_gap (GstH264Decoder * self, gint frame_num) +{ + GstH264DecoderPrivate *priv = self->priv; + const GstH264SPS *sps = priv->active_sps; + gint unused_short_term_frame_num; + + if (!sps) { + GST_ERROR_OBJECT (self, "No active sps"); + return FALSE; + } + + if (!sps->gaps_in_frame_num_value_allowed_flag) { + GST_WARNING_OBJECT (self, "Invalid frame num %d", frame_num); + return FALSE; + } + + GST_DEBUG_OBJECT (self, "Handling frame num gap %d -> %d", + priv->prev_ref_frame_num, frame_num); + + /* 7.4.3/7-23 */ + unused_short_term_frame_num = + (priv->prev_ref_frame_num + 1) % priv->max_frame_num; + while (unused_short_term_frame_num != frame_num) { + GstH264Picture *picture = gst_h264_picture_new (); + + if (!gst_h264_decoder_init_gap_picture (self, picture, + unused_short_term_frame_num)) + return FALSE; + + gst_h264_decoder_update_pic_nums (self, unused_short_term_frame_num); + + if (!gst_h264_decoder_finish_picture (self, picture)) { + GST_WARNING ("Failed to finish picture %p", picture); + return FALSE; + } + + unused_short_term_frame_num++; + unused_short_term_frame_num %= priv->max_frame_num; + } + + return TRUE; +} + +static gboolean +gst_h264_decoder_init_current_picture (GstH264Decoder * self) +{ + GstH264DecoderPrivate *priv = self->priv; + + if (!gst_h264_decoder_fill_picture_from_slice (self, &priv->current_slice, + priv->current_picture)) { + return FALSE; + } + + if (!gst_h264_decoder_calculate_poc (self, priv->current_picture)) + return FALSE; + + /* If the slice header indicates we will have to perform reference marking + * process after this picture is decoded, store required data for that + * purpose */ + if (priv->current_slice.header. + dec_ref_pic_marking.adaptive_ref_pic_marking_mode_flag) { + priv->current_picture->dec_ref_pic_marking = + priv->current_slice.header.dec_ref_pic_marking; + } + + return TRUE; +} + +static gboolean +gst_h264_decoder_start_current_picture (GstH264Decoder * self) +{ + GstH264DecoderClass *klass; + GstH264DecoderPrivate *priv = self->priv; + const GstH264SPS *sps; + gint frame_num; + gboolean ret = TRUE; + + g_assert (priv->current_picture != NULL); + g_assert (priv->active_sps != NULL); + g_assert (priv->active_pps != NULL); + + sps = priv->active_sps; + + priv->max_frame_num = 1 << (sps->log2_max_frame_num_minus4 + 4); + frame_num = priv->current_slice.header.frame_num; + if (priv->current_slice.nalu.idr_pic_flag) + priv->prev_ref_frame_num = 0; + + /* 7.4.3 */ + if (frame_num != priv->prev_ref_frame_num && + frame_num != (priv->prev_ref_frame_num + 1) % priv->max_frame_num) { + if (!gst_h264_decoder_handle_frame_num_gap (self, frame_num)) + return FALSE; + } + + if (!gst_h264_decoder_init_current_picture (self)) + return FALSE; + + gst_h264_decoder_update_pic_nums (self, frame_num); + + klass = GST_H264_DECODER_GET_CLASS (self); + if (klass->start_picture) + ret = klass->start_picture (self, priv->current_picture, + &priv->current_slice, priv->dpb); + + if (!ret) { + GST_ERROR_OBJECT (self, "subclass does not want to start picture"); + return FALSE; + } + + return TRUE; +} + +static gboolean +gst_h264_decoder_parse_slice (GstH264Decoder * self, GstH264NalUnit * nalu, + GstClockTime pts) +{ + GstH264DecoderPrivate *priv = self->priv; + GstH264ParserResult pres = GST_H264_PARSER_OK; + + memset (&priv->current_slice, 0, sizeof (GstH264Slice)); + + pres = gst_h264_parser_parse_slice_hdr (priv->parser, nalu, + &priv->current_slice.header, TRUE, TRUE); + + if (pres != GST_H264_PARSER_OK) { + GST_ERROR_OBJECT (self, "Failed to parse slice header, ret %d", pres); + memset (&priv->current_slice, 0, sizeof (GstH264Slice)); + + return FALSE; + } + + priv->current_slice.nalu = *nalu; + + if (!gst_h264_decoder_preprocess_slice (self, &priv->current_slice)) + return FALSE; + + priv->active_pps = priv->current_slice.header.pps; + priv->active_sps = priv->active_pps->sequence; + + if (!priv->current_picture) { + GstH264DecoderClass *klass = GST_H264_DECODER_GET_CLASS (self); + GstH264Picture *picture; + gboolean ret = TRUE; + + picture = gst_h264_picture_new (); + picture->pts = pts; + + if (klass->new_picture) + ret = klass->new_picture (self, picture); + + if (!ret) { + GST_ERROR_OBJECT (self, "subclass does not want accept new picture"); + gst_h264_picture_unref (picture); + return FALSE; + } + + priv->current_picture = picture; + + if (!gst_h264_decoder_start_current_picture (self)) { + GST_ERROR_OBJECT (self, "start picture failed"); + return FALSE; + } + } + + return gst_h264_decoder_decode_slice (self); +} + +static GstFlowReturn +gst_h264_decoder_parse_nal (GstH264Decoder * self, const guint8 * data, + gsize size, GstClockTime pts, gboolean at_eos, gsize * consumed_size) +{ + GstH264DecoderPrivate *priv = self->priv; + GstH264ParserResult pres; + GstH264NalUnit nalu; + gboolean ret = TRUE; + + *consumed_size = 0; + + if (priv->in_format == GST_H264_DECODER_FORMAT_AVC) { + if (priv->nal_length_size < 1 || priv->nal_length_size > 4) { + GST_ERROR_OBJECT (self, + "invalid nal length size %d", priv->nal_length_size); + return GST_FLOW_ERROR; + } + + pres = gst_h264_parser_identify_nalu_avc (priv->parser, + data, 0, size, priv->nal_length_size, &nalu); + + if (pres != GST_H264_PARSER_OK) { + GST_WARNING_OBJECT (self, "parsing avc nal ret %d", pres); + return GST_FLOW_ERROR; + } + } else { + if (size < 5) { + GST_DEBUG_OBJECT (self, "Too small data"); + return GST_VIDEO_DECODER_FLOW_NEED_DATA; + } + + pres = gst_h264_parser_identify_nalu (priv->parser, data, 0, size, &nalu); + + if (pres != GST_H264_PARSER_OK) { + if (pres == GST_H264_PARSER_NO_NAL_END) { + if (at_eos || priv->align == GST_H264_DECODER_ALIGN_AU) { + /* assume au boundary */ + } else { + return GST_VIDEO_DECODER_FLOW_NEED_DATA; + } + } else { + GST_WARNING_OBJECT (self, "parser ret %d", pres); + return GST_FLOW_ERROR; + } + } + } + + GST_LOG_OBJECT (self, "Parsed nal type: %d, offset %d, size %d", + nalu.type, nalu.offset, nalu.size); + + switch (nalu.type) { + case GST_H264_NAL_SPS: + ret = gst_h264_decoder_parse_sps (self, &nalu); + break; + case GST_H264_NAL_PPS: + ret = gst_h264_decoder_parse_pps (self, &nalu); + break; + case GST_H264_NAL_SLICE: + case GST_H264_NAL_SLICE_DPA: + case GST_H264_NAL_SLICE_DPB: + case GST_H264_NAL_SLICE_DPC: + case GST_H264_NAL_SLICE_IDR: + case GST_H264_NAL_SLICE_EXT: + ret = gst_h264_decoder_parse_slice (self, &nalu, pts); + break; + default: + break; + } + + if (consumed_size) + *consumed_size = nalu.offset + nalu.size; + + if (!ret) + return GST_FLOW_ERROR; + + return GST_FLOW_OK; +} + +static GstFlowReturn +gst_h264_decoder_parse (GstVideoDecoder * decoder, GstVideoCodecFrame * frame, + GstAdapter * adapter, gboolean at_eos) +{ + GstH264Decoder *self = GST_H264_DECODER (decoder); + GstH264DecoderPrivate *priv = self->priv; + GstFlowReturn ret = GST_FLOW_OK; + guint size; + const guint8 *data; + gsize consumed = 0; + + /* The return from have_frame() or output_picture() */ + priv->last_ret = GST_FLOW_OK; + + size = gst_adapter_available (adapter); + + data = (const guint8 *) gst_adapter_map (adapter, size); + ret = gst_h264_decoder_parse_nal (self, data, size, + gst_adapter_prev_pts (adapter, NULL), at_eos, &consumed); + gst_adapter_unmap (adapter); + + if (consumed) { + GST_TRACE_OBJECT (self, "consumed size %" G_GSIZE_FORMAT, consumed); + gst_video_decoder_add_to_frame (decoder, consumed); + } + + if (ret == GST_FLOW_ERROR) + goto error; + + /* When AU alginment and has no available input data more, + * finish current picture if any */ + if (priv->align == GST_H264_DECODER_ALIGN_AU && + !gst_adapter_available (adapter)) { + gst_h264_decoder_finish_current_picture (self); + } + + /* check last flow return again */ + if (ret == GST_FLOW_ERROR) + goto error; + + return priv->last_ret; + +error: + GST_VIDEO_DECODER_ERROR (self, 1, STREAM, DECODE, + ("Failed to decode data"), (NULL), ret); + + return ret; +} + +static void +gst_h264_decoder_format_from_caps (GstH264Decoder * self, GstCaps * caps, + GstH264DecoderFormat * format, GstH264DecoderAlign * align) +{ + if (format) + *format = GST_H264_DECODER_FORMAT_NONE; + + if (align) + *align = GST_H264_DECODER_ALIGN_NONE; + + if (!gst_caps_is_fixed (caps)) { + GST_WARNING_OBJECT (self, "Caps wasn't fixed"); + return; + } + + GST_DEBUG_OBJECT (self, "parsing caps: %" GST_PTR_FORMAT, caps); + + if (caps && gst_caps_get_size (caps) > 0) { + GstStructure *s = gst_caps_get_structure (caps, 0); + const gchar *str = NULL; + + if (format) { + if ((str = gst_structure_get_string (s, "stream-format"))) { + if (strcmp (str, "avc") == 0 || strcmp (str, "avc3") == 0) + *format = GST_H264_DECODER_FORMAT_AVC; + else if (strcmp (str, "byte-stream") == 0) + *format = GST_H264_DECODER_FORMAT_BYTE; + } + } + + if (align) { + if ((str = gst_structure_get_string (s, "alignment"))) { + if (strcmp (str, "au") == 0) + *align = GST_H264_DECODER_ALIGN_AU; + else if (strcmp (str, "nal") == 0) + *align = GST_H264_DECODER_ALIGN_NAL; + } + } + } +} + +static gboolean +gst_h264_decoder_set_format (GstVideoDecoder * decoder, + GstVideoCodecState * state) +{ + GstH264Decoder *self = GST_H264_DECODER (decoder); + GstH264DecoderPrivate *priv = self->priv; + + GST_DEBUG_OBJECT (decoder, "Set format"); + + if (self->input_state) + gst_video_codec_state_unref (self->input_state); + + self->input_state = gst_video_codec_state_ref (state); + + if (state->caps) { + GstStructure *str; + const GValue *codec_data_value; + GstH264DecoderFormat format; + GstH264DecoderAlign align; + + gst_h264_decoder_format_from_caps (self, state->caps, &format, &align); + + str = gst_caps_get_structure (state->caps, 0); + codec_data_value = gst_structure_get_value (str, "codec_data"); + + if (GST_VALUE_HOLDS_BUFFER (codec_data_value)) { + gst_buffer_replace (&priv->codec_data, + gst_value_get_buffer (codec_data_value)); + } else { + gst_buffer_replace (&priv->codec_data, NULL); + } + + if (format == GST_H264_DECODER_FORMAT_NONE) { + /* codec_data implies avc */ + if (codec_data_value != NULL) { + GST_WARNING_OBJECT (self, + "video/x-h264 caps with codec_data but no stream-format=avc"); + format = GST_H264_DECODER_FORMAT_AVC; + } else { + /* otherwise assume bytestream input */ + GST_WARNING_OBJECT (self, + "video/x-h264 caps without codec_data or stream-format"); + format = GST_H264_DECODER_FORMAT_BYTE; + } + } + + if (format == GST_H264_DECODER_FORMAT_AVC) { + /* AVC requires codec_data, AVC3 might have one and/or SPS/PPS inline */ + if (codec_data_value == NULL) { + /* Try it with size 4 anyway */ + priv->nal_length_size = 4; + GST_WARNING_OBJECT (self, + "avc format without codec data, assuming nal length size is 4"); + } + + /* AVC implies alignment=au */ + if (align == GST_H264_DECODER_ALIGN_NONE) + align = GST_H264_DECODER_ALIGN_AU; + } + + if (format == GST_H264_DECODER_FORMAT_BYTE) { + if (codec_data_value != NULL) { + GST_WARNING_OBJECT (self, "bytestream with codec data"); + } + } + + priv->in_format = format; + priv->align = align; + } + + if (priv->codec_data) { + GstMapInfo map; + + gst_buffer_map (priv->codec_data, &map, GST_MAP_READ); + gst_h264_decoder_parse_codec_data (self, map.data, map.size); + gst_buffer_unmap (priv->codec_data, &map); + } + + return TRUE; +} + +static gboolean +gst_h264_decoder_fill_picture_from_slice (GstH264Decoder * self, + const GstH264Slice * slice, GstH264Picture * picture) +{ + const GstH264SliceHdr *slice_hdr = &slice->header; + const GstH264PPS *pps; + const GstH264SPS *sps; + + pps = slice_hdr->pps; + if (!pps) { + GST_ERROR_OBJECT (self, "No pps in slice header"); + return FALSE; + } + + sps = pps->sequence; + if (!sps) { + GST_ERROR_OBJECT (self, "No sps in pps"); + return FALSE; + } + + picture->idr = slice->nalu.idr_pic_flag; + if (picture->idr) + picture->idr_pic_id = slice_hdr->idr_pic_id; + + if (slice_hdr->field_pic_flag) + picture->field = + slice_hdr->bottom_field_flag ? + GST_H264_PICTURE_FIELD_BOTTOM_FIELD : GST_H264_PICTURE_FILED_TOP_FIELD; + else + picture->field = GST_H264_PICTURE_FIELD_FRAME; + + if (picture->field != GST_H264_PICTURE_FIELD_FRAME) { + GST_FIXME ("Interlace video not supported"); + return FALSE; + } + + picture->nal_ref_idc = slice->nalu.ref_idc; + picture->ref = slice->nalu.ref_idc != 0; + + /* This assumes non-interlaced stream */ + picture->frame_num = picture->pic_num = slice_hdr->frame_num; + + picture->pic_order_cnt_type = sps->pic_order_cnt_type; + switch (picture->pic_order_cnt_type) { + case 0: + picture->pic_order_cnt_lsb = slice_hdr->pic_order_cnt_lsb; + picture->delta_pic_order_cnt_bottom = + slice_hdr->delta_pic_order_cnt_bottom; + break; + case 1: + picture->delta_pic_order_cnt0 = slice_hdr->delta_pic_order_cnt[0]; + picture->delta_pic_order_cnt1 = slice_hdr->delta_pic_order_cnt[1]; + break; + case 2: + break; + default: + g_assert_not_reached (); + return FALSE; + } + + return TRUE; +} + +static gboolean +gst_h264_decoder_calculate_poc (GstH264Decoder * self, GstH264Picture * picture) +{ + GstH264DecoderPrivate *priv = self->priv; + const GstH264SPS *sps = priv->active_sps; + + if (!sps) { + GST_ERROR_OBJECT (self, "No active SPS"); + return FALSE; + } + + switch (picture->pic_order_cnt_type) { + case 0:{ + /* See spec 8.2.1.1 */ + gint prev_pic_order_cnt_msb, prev_pic_order_cnt_lsb; + gint max_pic_order_cnt_lsb; + + if (picture->idr) { + prev_pic_order_cnt_msb = prev_pic_order_cnt_lsb = 0; + } else { + if (priv->prev_ref_has_memmgmnt5) { + if (priv->prev_ref_field != GST_H264_PICTURE_FIELD_BOTTOM_FIELD) { + prev_pic_order_cnt_msb = 0; + prev_pic_order_cnt_lsb = priv->prev_ref_top_field_order_cnt; + } else { + prev_pic_order_cnt_msb = 0; + prev_pic_order_cnt_lsb = 0; + } + } else { + prev_pic_order_cnt_msb = priv->prev_ref_pic_order_cnt_msb; + prev_pic_order_cnt_lsb = priv->prev_ref_pic_order_cnt_lsb; + } + } + + max_pic_order_cnt_lsb = 1 << (sps->log2_max_pic_order_cnt_lsb_minus4 + 4); + + if ((picture->pic_order_cnt_lsb < prev_pic_order_cnt_lsb) && + (prev_pic_order_cnt_lsb - picture->pic_order_cnt_lsb >= + max_pic_order_cnt_lsb / 2)) { + picture->pic_order_cnt_msb = + prev_pic_order_cnt_msb + max_pic_order_cnt_lsb; + } else if ((picture->pic_order_cnt_lsb > prev_pic_order_cnt_lsb) + && (picture->pic_order_cnt_lsb - prev_pic_order_cnt_lsb > + max_pic_order_cnt_lsb / 2)) { + picture->pic_order_cnt_msb = + prev_pic_order_cnt_msb - max_pic_order_cnt_lsb; + } else { + picture->pic_order_cnt_msb = prev_pic_order_cnt_msb; + } + + if (picture->field != GST_H264_PICTURE_FIELD_BOTTOM_FIELD) { + picture->top_field_order_cnt = + picture->pic_order_cnt_msb + picture->pic_order_cnt_lsb; + } + + if (picture->field != GST_H264_PICTURE_FILED_TOP_FIELD) { + if (picture->field == GST_H264_PICTURE_FIELD_FRAME) { + picture->bottom_field_order_cnt = + picture->top_field_order_cnt + + picture->delta_pic_order_cnt_bottom; + } else { + picture->bottom_field_order_cnt = + picture->pic_order_cnt_msb + picture->pic_order_cnt_lsb; + } + } + break; + } + + case 1:{ + gint abs_frame_num = 0; + gint expected_pic_order_cnt = 0; + gint i; + + /* See spec 8.2.1.2 */ + if (priv->prev_has_memmgmnt5) + priv->prev_frame_num_offset = 0; + + if (picture->idr) + picture->frame_num_offset = 0; + else if (priv->prev_frame_num > picture->frame_num) + picture->frame_num_offset = + priv->prev_frame_num_offset + priv->max_frame_num; + else + picture->frame_num_offset = priv->prev_frame_num_offset; + + if (sps->num_ref_frames_in_pic_order_cnt_cycle != 0) + abs_frame_num = picture->frame_num_offset + picture->frame_num; + else + abs_frame_num = 0; + + if (picture->nal_ref_idc == 0 && abs_frame_num > 0) + --abs_frame_num; + + if (abs_frame_num > 0) { + gint pic_order_cnt_cycle_cnt, frame_num_in_pic_order_cnt_cycle; + gint expected_delta_per_pic_order_cnt_cycle = 0; + + if (sps->num_ref_frames_in_pic_order_cnt_cycle == 0) { + GST_WARNING_OBJECT (self, + "Invalid num_ref_frames_in_pic_order_cnt_cycle in stream"); + return FALSE; + } + + pic_order_cnt_cycle_cnt = + (abs_frame_num - 1) / sps->num_ref_frames_in_pic_order_cnt_cycle; + frame_num_in_pic_order_cnt_cycle = + (abs_frame_num - 1) % sps->num_ref_frames_in_pic_order_cnt_cycle; + + for (i = 0; i < sps->num_ref_frames_in_pic_order_cnt_cycle; i++) { + expected_delta_per_pic_order_cnt_cycle += + sps->offset_for_ref_frame[i]; + } + + expected_pic_order_cnt = pic_order_cnt_cycle_cnt * + expected_delta_per_pic_order_cnt_cycle; + /* frame_num_in_pic_order_cnt_cycle is verified < 255 in parser */ + for (i = 0; i <= frame_num_in_pic_order_cnt_cycle; ++i) + expected_pic_order_cnt += sps->offset_for_ref_frame[i]; + } + + if (!picture->nal_ref_idc) + expected_pic_order_cnt += sps->offset_for_non_ref_pic; + + if (picture->field == GST_H264_PICTURE_FIELD_FRAME) { + picture->top_field_order_cnt = + expected_pic_order_cnt + picture->delta_pic_order_cnt0; + picture->bottom_field_order_cnt = picture->top_field_order_cnt + + sps->offset_for_top_to_bottom_field + picture->delta_pic_order_cnt1; + } else if (picture->field != GST_H264_PICTURE_FIELD_BOTTOM_FIELD) { + picture->top_field_order_cnt = + expected_pic_order_cnt + picture->delta_pic_order_cnt0; + } else { + picture->bottom_field_order_cnt = expected_pic_order_cnt + + sps->offset_for_top_to_bottom_field + picture->delta_pic_order_cnt0; + } + break; + } + + case 2:{ + gint temp_pic_order_cnt; + + /* See spec 8.2.1.3 */ + if (priv->prev_has_memmgmnt5) + priv->prev_frame_num_offset = 0; + + if (picture->idr) + picture->frame_num_offset = 0; + else if (priv->prev_frame_num > picture->frame_num) + picture->frame_num_offset = + priv->prev_frame_num_offset + priv->max_frame_num; + else + picture->frame_num_offset = priv->prev_frame_num_offset; + + if (picture->idr) { + temp_pic_order_cnt = 0; + } else if (!picture->nal_ref_idc) { + temp_pic_order_cnt = + 2 * (picture->frame_num_offset + picture->frame_num) - 1; + } else { + temp_pic_order_cnt = + 2 * (picture->frame_num_offset + picture->frame_num); + } + + if (picture->field == GST_H264_PICTURE_FIELD_FRAME) { + picture->top_field_order_cnt = temp_pic_order_cnt; + picture->bottom_field_order_cnt = temp_pic_order_cnt; + } else if (picture->field == GST_H264_PICTURE_FIELD_BOTTOM_FIELD) { + picture->bottom_field_order_cnt = temp_pic_order_cnt; + } else { + picture->top_field_order_cnt = temp_pic_order_cnt; + } + break; + } + + default: + GST_WARNING_OBJECT (self, + "Invalid pic_order_cnt_type: %d", sps->pic_order_cnt_type); + return FALSE; + } + + switch (picture->field) { + case GST_H264_PICTURE_FIELD_FRAME: + picture->pic_order_cnt = + MIN (picture->top_field_order_cnt, picture->bottom_field_order_cnt); + break; + case GST_H264_PICTURE_FILED_TOP_FIELD: + picture->pic_order_cnt = picture->top_field_order_cnt; + break; + case GST_H264_PICTURE_FIELD_BOTTOM_FIELD: + picture->pic_order_cnt = picture->bottom_field_order_cnt; + break; + default: + g_assert_not_reached (); + return FALSE; + } + + return TRUE; +} + +static void +gst_h264_decoder_do_output_picture (GstH264Decoder * self, + GstH264Picture * picture) +{ + GstH264DecoderPrivate *priv = self->priv; + GstH264DecoderClass *klass; + + picture->outputted = TRUE; + + if (picture->nonexisting) { + GST_DEBUG_OBJECT (self, "Skipping output, non-existing frame_num %d", + picture->frame_num); + return; + } + + if (picture->pic_order_cnt < priv->last_output_poc) { + GST_WARNING_OBJECT (self, + "Outputting out of order %d -> %d, likely a broken stream", + priv->last_output_poc, picture->pic_order_cnt); + } + + priv->last_output_poc = picture->pic_order_cnt; + + klass = GST_H264_DECODER_GET_CLASS (self); + + if (klass->output_picture) + priv->last_ret = klass->output_picture (self, picture); +} + +static gboolean +gst_h264_decoder_finish_current_picture (GstH264Decoder * self) +{ + GstH264DecoderPrivate *priv = self->priv; + GstH264DecoderClass *klass; + GstH264Picture *picture; + gboolean ret = TRUE; + + if (!priv->current_picture) + return TRUE; + + picture = priv->current_picture; + priv->current_picture = NULL; + + klass = GST_H264_DECODER_GET_CLASS (self); + + if (klass->end_picture) + ret = klass->end_picture (self, picture); + + gst_video_decoder_have_frame (GST_VIDEO_DECODER (self)); + + /* finish picture takes ownership of the picture */ + if (!gst_h264_decoder_finish_picture (self, picture)) { + GST_ERROR_OBJECT (self, "Failed to finish picture"); + return FALSE; + } + + return ret; +} + +static gint +poc_asc_compare (const GstH264Picture * a, const GstH264Picture * b) +{ + return a->pic_order_cnt > b->pic_order_cnt; +} + +static gboolean +gst_h264_decoder_output_all_remaining_pics (GstH264Decoder * self) +{ + GstH264DecoderPrivate *priv = self->priv; + GList *to_output = NULL; + GList *iter; + + gst_h264_dpb_get_pictures_not_outputted (priv->dpb, &to_output); + + to_output = g_list_sort (to_output, (GCompareFunc) poc_asc_compare); + + for (iter = to_output; iter; iter = g_list_next (iter)) { + GstH264Picture *picture = (GstH264Picture *) iter->data; + + GST_LOG_OBJECT (self, "Output picture %p (frame num %d, poc %d)", picture, + picture->frame_num, picture->pic_order_cnt); + gst_h264_decoder_do_output_picture (self, picture); + } + + if (to_output) + g_list_free_full (to_output, (GDestroyNotify) gst_h264_picture_unref); + + return TRUE; +} + +static gboolean +gst_h264_decoder_handle_memory_management_opt (GstH264Decoder * self, + GstH264Picture * picture) +{ + GstH264DecoderPrivate *priv = self->priv; + gint i; + + for (i = 0; i < G_N_ELEMENTS (picture->dec_ref_pic_marking.ref_pic_marking); + i++) { + GstH264RefPicMarking *ref_pic_marking = + &picture->dec_ref_pic_marking.ref_pic_marking[i]; + GstH264Picture *to_mark; + gint pic_num_x; + + switch (ref_pic_marking->memory_management_control_operation) { + case 0: + /* Normal end of operations' specification */ + return TRUE; + case 1: + /* Mark a short term reference picture as unused so it can be removed + * if outputted */ + pic_num_x = + picture->pic_num - (ref_pic_marking->difference_of_pic_nums_minus1 + + 1); + to_mark = gst_h264_dpb_get_short_ref_by_pic_num (priv->dpb, pic_num_x); + if (to_mark) { + to_mark->ref = FALSE; + gst_h264_picture_unref (to_mark); + } else { + GST_WARNING_OBJECT (self, "Invalid short term ref pic num to unmark"); + return FALSE; + } + break; + + case 2: + /* Mark a long term reference picture as unused so it can be removed + * if outputted */ + to_mark = gst_h264_dpb_get_long_ref_by_pic_num (priv->dpb, + ref_pic_marking->long_term_pic_num); + if (to_mark) { + to_mark->ref = FALSE; + gst_h264_picture_unref (to_mark); + } else { + GST_WARNING_OBJECT (self, "Invalid long term ref pic num to unmark"); + return FALSE; + } + break; + + case 3: + /* Mark a short term reference picture as long term reference */ + pic_num_x = + picture->pic_num - (ref_pic_marking->difference_of_pic_nums_minus1 + + 1); + to_mark = gst_h264_dpb_get_short_ref_by_pic_num (priv->dpb, pic_num_x); + if (to_mark) { + to_mark->long_term = TRUE; + to_mark->long_term_frame_idx = ref_pic_marking->long_term_frame_idx; + gst_h264_picture_unref (to_mark); + } else { + GST_WARNING_OBJECT (self, + "Invalid short term ref pic num to mark as long ref"); + return FALSE; + } + break; + + case 4:{ + GList *long_terms = NULL; + GList *iter; + + /* Unmark all reference pictures with long_term_frame_idx over new max */ + priv->max_long_term_frame_idx = + ref_pic_marking->max_long_term_frame_idx_plus1 - 1; + + gst_h264_dpb_get_pictures_long_term_ref (priv->dpb, &long_terms); + + for (iter = long_terms; iter; iter = g_list_next (iter)) { + GstH264Picture *long_term_picture = (GstH264Picture *) iter->data; + if (long_term_picture->long_term_frame_idx > + priv->max_long_term_frame_idx) + long_term_picture->ref = FALSE; + } + + if (long_terms) + g_list_free_full (long_terms, + (GDestroyNotify) gst_h264_picture_unref); + break; + } + + case 5: + /* Unmark all reference pictures */ + gst_h264_dpb_mark_all_non_ref (priv->dpb); + priv->max_long_term_frame_idx = -1; + picture->mem_mgmt_5 = TRUE; + break; + + case 6:{ + /* Replace long term reference pictures with current picture. + * First unmark if any existing with this long_term_frame_idx... */ + GList *long_terms = NULL; + GList *iter; + + gst_h264_dpb_get_pictures_long_term_ref (priv->dpb, &long_terms); + + for (iter = long_terms; iter; iter = g_list_next (iter)) { + GstH264Picture *long_term_picture = (GstH264Picture *) iter->data; + + if (long_term_picture->long_term_frame_idx == + ref_pic_marking->long_term_frame_idx) + long_term_picture->ref = FALSE; + } + + if (long_terms) + g_list_free_full (long_terms, + (GDestroyNotify) gst_h264_picture_unref); + + /* and mark the current one instead */ + picture->ref = TRUE; + picture->long_term = TRUE; + picture->long_term_frame_idx = ref_pic_marking->long_term_frame_idx; + break; + } + + default: + g_assert_not_reached (); + break; + } + } + + return TRUE; +} + +static gboolean +gst_h264_decoder_sliding_window_picture_marking (GstH264Decoder * self) +{ + GstH264DecoderPrivate *priv = self->priv; + const GstH264SPS *sps = priv->active_sps; + gint num_ref_pics; + gint max_num_ref_frames; + + if (!sps) { + GST_ERROR_OBJECT (self, "No active sps"); + return FALSE; + } + + /* 8.2.5.3. Ensure the DPB doesn't overflow by discarding the oldest picture */ + num_ref_pics = gst_h264_dpb_num_ref_pictures (priv->dpb); + max_num_ref_frames = MAX (1, sps->num_ref_frames); + + if (num_ref_pics > max_num_ref_frames) { + GST_WARNING_OBJECT (self, + "num_ref_pics %d is larger than allowed maximum %d", + num_ref_pics, max_num_ref_frames); + return FALSE; + } + + if (num_ref_pics == max_num_ref_frames) { + /* Max number of reference pics reached, need to remove one of the short + * term ones. Find smallest frame_num_wrap short reference picture and mark + * it as unused */ + GstH264Picture *to_unmark = + gst_h264_dpb_get_lowest_frame_num_short_ref (priv->dpb); + + if (!to_unmark) { + GST_WARNING_OBJECT (self, "Could not find a short ref picture to unmark"); + return FALSE; + } + + to_unmark->ref = FALSE; + gst_h264_picture_unref (to_unmark); + } + + return TRUE; +} + +/* This method ensures that DPB does not overflow, either by removing + * reference pictures as specified in the stream, or using a sliding window + * procedure to remove the oldest one. + * It also performs marking and unmarking pictures as reference. + * See spac 8.2.5.1 */ +static gboolean +gst_h264_decoder_reference_picture_marking (GstH264Decoder * self, + GstH264Picture * picture) +{ + GstH264DecoderPrivate *priv = self->priv; + + /* If the current picture is an IDR, all reference pictures are unmarked */ + if (picture->idr) { + gst_h264_dpb_mark_all_non_ref (priv->dpb); + + if (picture->dec_ref_pic_marking.long_term_reference_flag) { + picture->long_term = TRUE; + picture->long_term_frame_idx = 0; + priv->max_long_term_frame_idx = 0; + } else { + picture->long_term = FALSE; + priv->max_long_term_frame_idx = -1; + } + + return TRUE; + } + + /* Not an IDR. If the stream contains instructions on how to discard pictures + * from DPB and how to mark/unmark existing reference pictures, do so. + * Otherwise, fall back to default sliding window process */ + if (picture->dec_ref_pic_marking.adaptive_ref_pic_marking_mode_flag) { + return gst_h264_decoder_handle_memory_management_opt (self, picture); + } + + return gst_h264_decoder_sliding_window_picture_marking (self); +} + +static gboolean +gst_h264_decoder_finish_picture (GstH264Decoder * self, + GstH264Picture * picture) +{ + GstH264DecoderPrivate *priv = self->priv; + GList *not_outputted = NULL; + guint num_remaining; + GList *iter; +#ifndef GST_DISABLE_GST_DEBUG + gint i; +#endif + + /* Finish processing the picture. + * Start by storing previous picture data for later use */ + if (picture->ref) { + gst_h264_decoder_reference_picture_marking (self, picture); + priv->prev_ref_has_memmgmnt5 = picture->mem_mgmt_5; + priv->prev_ref_top_field_order_cnt = picture->top_field_order_cnt; + priv->prev_ref_pic_order_cnt_msb = picture->pic_order_cnt_msb; + priv->prev_ref_pic_order_cnt_lsb = picture->pic_order_cnt_lsb; + priv->prev_ref_field = picture->field; + priv->prev_ref_frame_num = picture->frame_num; + } + + priv->prev_frame_num = picture->frame_num; + priv->prev_has_memmgmnt5 = picture->mem_mgmt_5; + priv->prev_frame_num_offset = picture->frame_num_offset; + + /* Remove unused (for reference or later output) pictures from DPB, marking + * them as such */ + gst_h264_dpb_delete_unused (priv->dpb); + + GST_LOG_OBJECT (self, + "Finishing picture %p (frame_num %d, poc %d), entries in DPB %d", + picture, picture->frame_num, picture->pic_order_cnt, + gst_h264_dpb_get_size (priv->dpb)); + + /* The ownership of pic will either be transferred to DPB - if the picture is + * still needed (for output and/or reference) - or we will release it + * immediately if we manage to output it here and won't have to store it for + * future reference */ + + /* Get all pictures that haven't been outputted yet */ + gst_h264_dpb_get_pictures_not_outputted (priv->dpb, ¬_outputted); + /* Include the one we've just decoded */ + not_outputted = g_list_append (not_outputted, picture); + + /* for debugging */ +#ifndef GST_DISABLE_GST_DEBUG + GST_TRACE_OBJECT (self, "Before sorting not outputted list"); + i = 0; + for (iter = not_outputted; iter; iter = g_list_next (iter)) { + GstH264Picture *tmp = (GstH264Picture *) iter->data; + + GST_TRACE_OBJECT (self, + "\t%dth picture %p (poc %d)", i, tmp, tmp->pic_order_cnt); + i++; + } +#endif + + /* Sort in output order */ + not_outputted = g_list_sort (not_outputted, (GCompareFunc) poc_asc_compare); + +#ifndef GST_DISABLE_GST_DEBUG + GST_TRACE_OBJECT (self, + "After sorting not outputted list in poc ascending order"); + i = 0; + for (iter = not_outputted; iter; iter = g_list_next (iter)) { + GstH264Picture *tmp = (GstH264Picture *) iter->data; + + GST_TRACE_OBJECT (self, + "\t%dth picture %p (poc %d)", i, tmp, tmp->pic_order_cnt); + i++; + } +#endif + + /* Try to output as many pictures as we can. A picture can be output, + * if the number of decoded and not yet outputted pictures that would remain + * in DPB afterwards would at least be equal to max_num_reorder_frames. + * If the outputted picture is not a reference picture, it doesn't have + * to remain in the DPB and can be removed */ + iter = not_outputted; + num_remaining = g_list_length (not_outputted); + + while (num_remaining > priv->max_num_reorder_frames || + /* If the condition below is used, this is an invalid stream. We should + * not be forced to output beyond max_num_reorder_frames in order to + * make room in DPB to store the current picture (if we need to do so). + * However, if this happens, ignore max_num_reorder_frames and try + * to output more. This may cause out-of-order output, but is not + * fatal, and better than failing instead */ + ((gst_h264_dpb_is_full (priv->dpb) && (!picture->outputted + || picture->ref)) + && num_remaining)) { + GstH264Picture *to_output = (GstH264Picture *) iter->data; + + if (num_remaining <= priv->max_num_reorder_frames) { + GST_WARNING_OBJECT (self, + "Invalid stream, max_num_reorder_frames not preserved"); + } + + GST_LOG_OBJECT (self, + "Output picture %p (frame num %d)", to_output, to_output->frame_num); + gst_h264_decoder_do_output_picture (self, to_output); + if (!to_output->ref) { + /* Current picture hasn't been inserted into DPB yet, so don't remove it + * if we managed to output it immediately */ + gint outputted_poc = to_output->pic_order_cnt; + if (outputted_poc != picture->pic_order_cnt) + gst_h264_dpb_delete_by_poc (priv->dpb, outputted_poc); + } + + iter = g_list_next (iter); + num_remaining--; + } + + /* If we haven't managed to output the picture that we just decoded, or if + * it's a reference picture, we have to store it in DPB */ + if (!picture->outputted || picture->ref) { + if (gst_h264_dpb_is_full (priv->dpb)) { + /* If we haven't managed to output anything to free up space in DPB + * to store this picture, it's an error in the stream */ + GST_WARNING_OBJECT (self, "Could not free up space in DPB"); + return FALSE; + } + + GST_TRACE_OBJECT (self, + "Put picture %p (outputted %d, ref %d, frame num %d, poc %d) to dpb", + picture, picture->outputted, picture->ref, picture->frame_num, + picture->pic_order_cnt); + gst_h264_dpb_add (priv->dpb, gst_h264_picture_ref (picture)); + } + + if (not_outputted) + g_list_free_full (not_outputted, (GDestroyNotify) gst_h264_picture_unref); + + return TRUE; +} + +static gboolean +gst_h264_decoder_update_max_num_reorder_frames (GstH264Decoder * self, + GstH264SPS * sps) +{ + GstH264DecoderPrivate *priv = self->priv; + + if (sps->vui_parameters_present_flag + && sps->vui_parameters.bitstream_restriction_flag) { + priv->max_num_reorder_frames = sps->vui_parameters.num_reorder_frames; + if (priv->max_num_reorder_frames > + gst_h264_dpb_get_max_num_pics (priv->dpb)) { + GST_WARNING + ("max_num_reorder_frames present, but larger than MaxDpbFrames (%d > %d)", + (gint) priv->max_num_reorder_frames, + gst_h264_dpb_get_max_num_pics (priv->dpb)); + + priv->max_num_reorder_frames = 0; + return FALSE; + } + + return TRUE; + } + + /* max_num_reorder_frames not present, infer from profile/constraints + * (see VUI semantics in spec) */ + if (sps->constraint_set3_flag) { + switch (sps->profile_idc) { + case 44: + case 86: + case 100: + case 110: + case 122: + case 244: + priv->max_num_reorder_frames = 0; + break; + default: + priv->max_num_reorder_frames = + gst_h264_dpb_get_max_num_pics (priv->dpb); + break; + } + } else { + priv->max_num_reorder_frames = gst_h264_dpb_get_max_num_pics (priv->dpb); + } + + return TRUE; +} + +typedef enum +{ + GST_H264_LEVEL_L1 = 10, + GST_H264_LEVEL_L1B = 9, + GST_H264_LEVEL_L1_1 = 11, + GST_H264_LEVEL_L1_2 = 12, + GST_H264_LEVEL_L1_3 = 13, + GST_H264_LEVEL_L2_0 = 20, + GST_H264_LEVEL_L2_1 = 21, + GST_H264_LEVEL_L2_2 = 22, + GST_H264_LEVEL_L3 = 30, + GST_H264_LEVEL_L3_1 = 31, + GST_H264_LEVEL_L3_2 = 32, + GST_H264_LEVEL_L4 = 40, + GST_H264_LEVEL_L4_1 = 41, + GST_H264_LEVEL_L4_2 = 42, + GST_H264_LEVEL_L5 = 50, + GST_H264_LEVEL_L5_1 = 51, + GST_H264_LEVEL_L5_2 = 52, + GST_H264_LEVEL_L6 = 60, + GST_H264_LEVEL_L6_1 = 61, + GST_H264_LEVEL_L6_2 = 62, +} GstD3D11H264Level; + +typedef struct +{ + GstD3D11H264Level level; + + guint32 max_mbps; + guint32 max_fs; + guint32 max_dpb_mbs; + guint32 max_main_br; +} LevelLimits; + +static const LevelLimits level_limits_map[] = { + {GST_H264_LEVEL_L1, 1485, 99, 396, 64}, + {GST_H264_LEVEL_L1B, 1485, 99, 396, 128}, + {GST_H264_LEVEL_L1_1, 3000, 396, 900, 192}, + {GST_H264_LEVEL_L1_2, 6000, 396, 2376, 384}, + {GST_H264_LEVEL_L1_3, 11800, 396, 2376, 768}, + {GST_H264_LEVEL_L2_0, 11880, 396, 2376, 2000}, + {GST_H264_LEVEL_L2_1, 19800, 792, 4752, 4000}, + {GST_H264_LEVEL_L2_2, 20250, 1620, 8100, 4000}, + {GST_H264_LEVEL_L3, 40500, 1620, 8100, 10000}, + {GST_H264_LEVEL_L3_1, 108000, 3600, 18000, 14000}, + {GST_H264_LEVEL_L3_2, 216000, 5120, 20480, 20000}, + {GST_H264_LEVEL_L4, 245760, 8192, 32768, 20000}, + {GST_H264_LEVEL_L4_1, 245760, 8192, 32768, 50000}, + {GST_H264_LEVEL_L4_2, 522240, 8704, 34816, 50000}, + {GST_H264_LEVEL_L5, 589824, 22080, 110400, 135000}, + {GST_H264_LEVEL_L5_1, 983040, 36864, 184320, 240000}, + {GST_H264_LEVEL_L5_2, 2073600, 36864, 184320, 240000}, + {GST_H264_LEVEL_L6, 4177920, 139264, 696320, 240000}, + {GST_H264_LEVEL_L6_1, 8355840, 139264, 696320, 480000}, + {GST_H264_LEVEL_L6_2, 16711680, 139264, 696320, 800000} +}; + +static gint +h264_level_to_max_dpb_mbs (GstD3D11H264Level level) +{ + gint i; + for (i = 0; i < G_N_ELEMENTS (level_limits_map); i++) { + if (level == level_limits_map[i].level) + return level_limits_map[i].max_dpb_mbs; + } + + return 0; +} + +static gboolean +gst_h264_decoder_process_sps (GstH264Decoder * self, GstH264SPS * sps) +{ + GstH264DecoderPrivate *priv = self->priv; + guint8 level; + gint max_dpb_mbs; + gint width_mb, height_mb; + gint max_dpb_frames; + gint max_dpb_size; + gint prev_max_dpb_size; + + if (sps->frame_mbs_only_flag == 0) { + GST_FIXME_OBJECT (self, "frame_mbs_only_flag != 1 not supported"); + return FALSE; + } + + /* Spec A.3.1 and A.3.2 + * For Baseline, Constrained Baseline and Main profile, the indicated level is + * Level 1b if level_idc is equal to 11 and constraint_set3_flag is equal to 1 + */ + level = sps->level_idc; + if (level == 11 && (sps->profile_idc == 66 || sps->profile_idc == 77) && + sps->constraint_set3_flag) { + /* Leel 1b */ + level = 9; + } + + max_dpb_mbs = h264_level_to_max_dpb_mbs ((GstD3D11H264Level) level); + if (!max_dpb_mbs) + return FALSE; + + width_mb = sps->width / 16; + height_mb = sps->height / 16; + + max_dpb_frames = MIN (max_dpb_mbs / (width_mb * height_mb), + GST_H264_DPB_MAX_SIZE); + + max_dpb_size = MAX (max_dpb_frames, + MAX (sps->num_ref_frames, sps->vui_parameters.max_dec_frame_buffering)); + + prev_max_dpb_size = gst_h264_dpb_get_max_num_pics (priv->dpb); + if (priv->width != sps->width || priv->height != sps->height || + prev_max_dpb_size != max_dpb_size) { + GstH264DecoderClass *klass = GST_H264_DECODER_GET_CLASS (self); + + GST_DEBUG_OBJECT (self, + "SPS updated, resolution: %dx%d -> %dx%d, dpb size: %d -> %d", + priv->width, priv->height, sps->width, sps->height, + prev_max_dpb_size, max_dpb_size); + + if (!gst_h264_decoder_flush (GST_VIDEO_DECODER (self))) + return FALSE; + + g_assert (klass->new_sequence); + + if (!klass->new_sequence (self, sps)) { + GST_ERROR_OBJECT (self, "subclass does not want accept new sequence"); + return FALSE; + } + + priv->width = sps->width; + priv->height = sps->height; + + gst_h264_dpb_set_max_num_pics (priv->dpb, max_dpb_size); + } + + GST_DEBUG_OBJECT (self, "Set DPB max size %d", max_dpb_size); + + return gst_h264_decoder_update_max_num_reorder_frames (self, sps); +} + +static gboolean +gst_h264_decoder_init_gap_picture (GstH264Decoder * self, + GstH264Picture * picture, gint frame_num) +{ + picture->nonexisting = TRUE; + picture->nal_ref_idc = 1; + picture->frame_num = picture->pic_num = frame_num; + picture->dec_ref_pic_marking.adaptive_ref_pic_marking_mode_flag = FALSE; + picture->ref = TRUE; + picture->dec_ref_pic_marking.long_term_reference_flag = FALSE; + picture->field = GST_H264_PICTURE_FIELD_FRAME; + + return gst_h264_decoder_calculate_poc (self, picture); +} + +static gboolean +gst_h264_decoder_decode_slice (GstH264Decoder * self) +{ + GstH264DecoderClass *klass = GST_H264_DECODER_GET_CLASS (self); + GstH264DecoderPrivate *priv = self->priv; + GstH264Slice *slice = &priv->current_slice; + GstH264Picture *picture = priv->current_picture; + + if (!picture) { + GST_ERROR_OBJECT (self, "No current picture"); + return FALSE; + } + + if (slice->header.field_pic_flag == 0) + priv->max_pic_num = priv->max_frame_num; + else + priv->max_pic_num = 2 * priv->max_frame_num; + + g_assert (klass->decode_slice); + + return klass->decode_slice (self, picture, slice); +} diff --git a/sys/d3d11/gsth264decoder.h b/sys/d3d11/gsth264decoder.h new file mode 100644 index 0000000000..7968b41b05 --- /dev/null +++ b/sys/d3d11/gsth264decoder.h @@ -0,0 +1,112 @@ +/* GStreamer + * Copyright (C) 2019 Seungha Yang + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifndef __GST_H264_DECODER_H__ +#define __GST_H264_DECODER_H__ + +#include +#include +#include +#include "gsth264picture.h" + +G_BEGIN_DECLS + +#define GST_TYPE_H264_DECODER (gst_h264_decoder_get_type()) +#define GST_H264_DECODER(obj) (G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_H264_DECODER,GstH264Decoder)) +#define GST_H264_DECODER_CLASS(klass) (G_TYPE_CHECK_CLASS_CAST((klass),GST_TYPE_H264_DECODER,GstH264DecoderClass)) +#define GST_H264_DECODER_GET_CLASS(obj) (G_TYPE_INSTANCE_GET_CLASS((obj),GST_TYPE_H264_DECODER,GstH264DecoderClass)) +#define GST_IS_H264_DECODER(obj) (G_TYPE_CHECK_INSTANCE_TYPE((obj),GST_TYPE_H264_DECODER)) +#define GST_IS_H264_DECODER_CLASS(klass) (G_TYPE_CHECK_CLASS_TYPE((klass),GST_TYPE_H264_DECODER)) +#define GST_H264_DECODER_CAST(obj) ((GstH264Decoder*)obj) + +typedef struct _GstH264Decoder GstH264Decoder; +typedef struct _GstH264DecoderClass GstH264DecoderClass; +typedef struct _GstH264DecoderPrivate GstH264DecoderPrivate; + +/** + * GstH264Decoder: + * + * The opaque #GstH264Decoder data structure. + */ +struct _GstH264Decoder +{ + /*< private >*/ + GstVideoDecoder parent; + + /*< protected >*/ + GstVideoCodecState * input_state; + + /*< private >*/ + GstH264DecoderPrivate *priv; + gpointer padding[GST_PADDING_LARGE]; +}; + +/** + * GstH264DecoderClass: + * @new_sequence: Notifies subclass of SPS update + * @new_picture: Optional. + * Called whenever new #GstH264Picture is created. + * Subclass can set implementation specific user data + * on the #GstH264Picture via gst_h264_picture_set_user_data() + * @output_picture: Optional. + * Called just before gst_video_decoder_have_frame(). + * Subclass should be prepared for handle_frame() + * @start_picture: Optional. + * Called per one #GstH264Picture to notify subclass to prepare + * decoding process for the #GstH264Picture + * @decode_slice: Provides per slice data with parsed slice header and + * required raw bitstream for subclass to decode it + * @end_picture: Optional. + * Called per one #GstH264Picture to notify subclass to finish + * decoding process for the #GstH264Picture + */ +struct _GstH264DecoderClass +{ + GstVideoDecoderClass parent_class; + + gboolean (*new_sequence) (GstH264Decoder * decoder, + const GstH264SPS * sps); + + gboolean (*new_picture) (GstH264Decoder * decoder, + GstH264Picture * picture); + + GstFlowReturn (*output_picture) (GstH264Decoder * decoder, + GstH264Picture * picture); + + gboolean (*start_picture) (GstH264Decoder * decoder, + GstH264Picture * picture, + GstH264Slice * slice, + GstH264Dpb * dpb); + + gboolean (*decode_slice) (GstH264Decoder * decoder, + GstH264Picture * picture, + GstH264Slice * slice); + + gboolean (*end_picture) (GstH264Decoder * decoder, + GstH264Picture * picture); + + /*< private >*/ + gpointer padding[GST_PADDING_LARGE]; +}; + +GType gst_h264_decoder_get_type (void); + +G_END_DECLS + +#endif /* __GST_H264_DECODER_H__ */ diff --git a/sys/d3d11/gsth264picture.c b/sys/d3d11/gsth264picture.c new file mode 100644 index 0000000000..7110a91c1b --- /dev/null +++ b/sys/d3d11/gsth264picture.c @@ -0,0 +1,510 @@ +/* GStreamer + * Copyright (C) 2019 Seungha Yang + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include "gsth264picture.h" + +GST_DEBUG_CATEGORY_EXTERN (gst_d3d11_h264_dec_debug); +#define GST_CAT_DEFAULT gst_d3d11_h264_dec_debug + +GST_DEFINE_MINI_OBJECT_TYPE (GstH264Picture, gst_h264_picture); + +static void +_gst_h264_picture_free (GstH264Picture * picture) +{ + if (picture->notify) + picture->notify (picture->user_data); + + g_free (picture); +} + +/** + * gst_h264_picture_new: + * + * Create new #GstH264Picture + * + * Returns: a new #GstH264Picture + */ +GstH264Picture * +gst_h264_picture_new (void) +{ + GstH264Picture *pic; + + pic = g_new0 (GstH264Picture, 1); + + pic->pts = GST_CLOCK_TIME_NONE; + pic->top_field_order_cnt = G_MAXINT32; + pic->bottom_field_order_cnt = G_MAXINT32; + pic->field = GST_H264_PICTURE_FIELD_FRAME; + + gst_mini_object_init (GST_MINI_OBJECT_CAST (pic), 0, + GST_TYPE_H264_PICTURE, NULL, NULL, + (GstMiniObjectFreeFunction) _gst_h264_picture_free); + + return pic; +} + +/** + * gst_h264_picture_set_user_data: + * @picture: a #GstH264Picture + * @user_data: private data + * @notify: (closure user_data): a #GDestroyNotify + * + * Sets @user_data on the picture and the #GDestroyNotify that will be called when + * the picture is freed. + * + * If a @user_data was previously set, then the previous set @notify will be called + * before the @user_data is replaced. + */ +void +gst_h264_picture_set_user_data (GstH264Picture * picture, gpointer user_data, + GDestroyNotify notify) +{ + g_return_if_fail (GST_IS_H264_PICTURE (picture)); + + if (picture->notify) + picture->notify (picture->user_data); + + picture->user_data = user_data; + picture->notify = notify; +} + +/** + * gst_h264_picture_get_user_data: + * @picture: a #GstH264Picture + * + * Gets private data set on the picture via + * gst_h264_picture_set_user_data() previously. + * + * Returns: (transfer none): The previously set user_data + */ +gpointer +gst_h264_picture_get_user_data (GstH264Picture * picture) +{ + return picture->user_data; +} + +struct _GstH264Dpb +{ + GArray *pic_list; + gint max_num_pics; +}; + +/** + * gst_h264_dpb_new: + * + * Create new #GstH264Dpb + * + * Returns: a new #GstH264Dpb + */ +GstH264Dpb * +gst_h264_dpb_new (void) +{ + GstH264Dpb *dpb; + + dpb = g_new0 (GstH264Dpb, 1); + + dpb->pic_list = + g_array_sized_new (FALSE, TRUE, sizeof (GstH264Picture *), + GST_H264_DPB_MAX_SIZE); + g_array_set_clear_func (dpb->pic_list, + (GDestroyNotify) gst_h264_picture_clear); + + return dpb; +} + +/** + * gst_h264_dpb_set_max_num_pics: + * @dpb: a #GstH264Dpb + * @max_num_pics: the maximum number of picture + * + * Set the number of maximum allowed pictures to store + */ +void +gst_h264_dpb_set_max_num_pics (GstH264Dpb * dpb, gint max_num_pics) +{ + g_return_if_fail (dpb != NULL); + + dpb->max_num_pics = max_num_pics; +} + +/** + * gst_h264_dpb_get_max_num_pics: + * @dpb: a #GstH264Dpb + * + * Returns: the number of maximum pictures + */ +gint +gst_h264_dpb_get_max_num_pics (GstH264Dpb * dpb) +{ + g_return_val_if_fail (dpb != NULL, 0); + + return dpb->max_num_pics; +} + +/** + * gst_h264_dpb_free: + * @dpb: a #GstH264Dpb to free + * + * Free the @dpb + */ +void +gst_h264_dpb_free (GstH264Dpb * dpb) +{ + g_return_if_fail (dpb != NULL); + + gst_h264_dpb_clear (dpb); + g_free (dpb); +} + +/** + * gst_h264_dpb_clear: + * @dpb: a #GstH264Dpb + * + * Clear all stored #GstH264Picture + */ +void +gst_h264_dpb_clear (GstH264Dpb * dpb) +{ + g_return_if_fail (dpb != NULL); + + g_array_set_size (dpb->pic_list, 0); +} + +/** + * gst_h264_dpb_add: + * @dpb: a #GstH264Dpb + * @picture: (transfer full): a #GstH264Picture + * + * Store the @picture + */ +void +gst_h264_dpb_add (GstH264Dpb * dpb, GstH264Picture * picture) +{ + g_return_if_fail (dpb != NULL); + g_return_if_fail (GST_IS_H264_PICTURE (picture)); + + g_array_append_val (dpb->pic_list, picture); +} + +/** + * gst_h264_dpb_delete_unused: + * @dpb: a #GstH264Dpb + * + * Delete already outputted and not referenced all pictures from dpb + */ +void +gst_h264_dpb_delete_unused (GstH264Dpb * dpb) +{ + gint i; + + g_return_if_fail (dpb != NULL); + + for (i = 0; i < dpb->pic_list->len; i++) { + GstH264Picture *picture = + g_array_index (dpb->pic_list, GstH264Picture *, i); + + if (picture->outputted && !picture->ref) { + GST_TRACE ("remove picture %p (frame num %d) from dpb", + picture, picture->frame_num); + g_array_remove_index (dpb->pic_list, i); + i--; + } + } +} + +/** + * gst_h264_dpb_delete_by_poc: + * @dpb: a #GstH264Dpb + * @poc: a poc of #GstH264Picture to remove + * + * Delete a #GstH264Dpb by @poc + */ +void +gst_h264_dpb_delete_by_poc (GstH264Dpb * dpb, gint poc) +{ + gint i; + + g_return_if_fail (dpb != NULL); + + for (i = 0; i < dpb->pic_list->len; i++) { + GstH264Picture *picture = + g_array_index (dpb->pic_list, GstH264Picture *, i); + + if (picture->pic_order_cnt == poc) { + g_array_remove_index (dpb->pic_list, i); + return; + } + } + + GST_WARNING ("Couldn't find picture with poc %d", poc); +} + +/** + * gst_h264_dpb_num_ref_pictures: + * @dpb: a #GstH264Dpb + * + * Returns: The number of referenced pictures + */ +gint +gst_h264_dpb_num_ref_pictures (GstH264Dpb * dpb) +{ + gint i; + gint ret = 0; + + g_return_val_if_fail (dpb != NULL, -1); + + for (i = 0; i < dpb->pic_list->len; i++) { + GstH264Picture *picture = + g_array_index (dpb->pic_list, GstH264Picture *, i); + + if (picture->ref) + ret++; + } + + return ret; +} + +/** + * gst_h264_dpb_mark_all_non_ref: + * @dpb: a #GstH264Dpb + * + * Mark all pictures are not referenced + */ +void +gst_h264_dpb_mark_all_non_ref (GstH264Dpb * dpb) +{ + gint i; + + g_return_if_fail (dpb != NULL); + + for (i = 0; i < dpb->pic_list->len; i++) { + GstH264Picture *picture = + g_array_index (dpb->pic_list, GstH264Picture *, i); + + picture->ref = FALSE; + } +} + +/** + * gst_h264_dpb_get_short_ref_by_pic_num: + * @dpb: a #GstH264Dpb + * @pic_num: a picture number + * + * Find a short term reference picture which has matching picture number + * + * Returns: (nullable) (transfer full): a #GstH264Picture + */ +GstH264Picture * +gst_h264_dpb_get_short_ref_by_pic_num (GstH264Dpb * dpb, gint pic_num) +{ + gint i; + + g_return_val_if_fail (dpb != NULL, NULL); + + for (i = 0; i < dpb->pic_list->len; i++) { + GstH264Picture *picture = + g_array_index (dpb->pic_list, GstH264Picture *, i); + + if (picture->ref && !picture->long_term && picture->pic_num == pic_num) + return gst_h264_picture_ref (picture); + } + + GST_WARNING ("No short term reference picture for %d", pic_num); + + return NULL; +} + +/** + * gst_h264_dpb_get_long_ref_by_pic_num: + * @dpb: a #GstH264Dpb + * @pic_num: a picture number + * + * Find a long term reference picture which has matching picture number + * + * Returns: (nullable) (transfer full): a #GstH264Picture + */ +GstH264Picture * +gst_h264_dpb_get_long_ref_by_pic_num (GstH264Dpb * dpb, gint pic_num) +{ + gint i; + + g_return_val_if_fail (dpb != NULL, NULL); + + for (i = 0; i < dpb->pic_list->len; i++) { + GstH264Picture *picture = + g_array_index (dpb->pic_list, GstH264Picture *, i); + + if (picture->ref && picture->long_term && picture->pic_num == pic_num) + return gst_h264_picture_ref (picture); + } + + GST_WARNING ("No long term reference picture for %d", pic_num); + + return NULL; +} + +/** + * gst_h264_dpb_get_lowest_frame_num_short_ref: + * @dpb: a #GstH264Dpb + * @pic_num: a picture number + * + * Find a short term reference picture which has the lowest frame_num_wrap + * + * Returns: (transfer full): a #GstH264Picture + */ +GstH264Picture * +gst_h264_dpb_get_lowest_frame_num_short_ref (GstH264Dpb * dpb) +{ + gint i; + GstH264Picture *ret = NULL; + + g_return_val_if_fail (dpb != NULL, NULL); + + for (i = 0; i < dpb->pic_list->len; i++) { + GstH264Picture *picture = + g_array_index (dpb->pic_list, GstH264Picture *, i); + + if (picture->ref && !picture->long_term && + (!ret || picture->frame_num_wrap < ret->frame_num_wrap)) + ret = picture; + } + + if (ret) + gst_h264_picture_ref (ret); + + return ret; +} + +/** + * gst_h264_dpb_get_pictures_not_outputted: + * @dpb: a #GstH264Dpb + * @out: (out): a list of #GstH264Dpb + * + * Retrieve all not-outputted pictures from @dpb + */ +void +gst_h264_dpb_get_pictures_not_outputted (GstH264Dpb * dpb, GList ** out) +{ + gint i; + + g_return_if_fail (dpb != NULL); + g_return_if_fail (out != NULL); + + for (i = 0; i < dpb->pic_list->len; i++) { + GstH264Picture *picture = + g_array_index (dpb->pic_list, GstH264Picture *, i); + + if (!picture->outputted) + *out = g_list_append (*out, gst_h264_picture_ref (picture)); + } +} + +/** + * gst_h264_dpb_get_pictures_short_term_ref: + * @dpb: a #GstH264Dpb + * @out: (out): a list of #GstH264Dpb + * + * Retrieve all short-term reference pictures from @dpb + */ +void +gst_h264_dpb_get_pictures_short_term_ref (GstH264Dpb * dpb, GList ** out) +{ + gint i; + + g_return_if_fail (dpb != NULL); + g_return_if_fail (out != NULL); + + for (i = 0; i < dpb->pic_list->len; i++) { + GstH264Picture *picture = + g_array_index (dpb->pic_list, GstH264Picture *, i); + + if (picture->ref && !picture->long_term) + *out = g_list_append (*out, gst_h264_picture_ref (picture)); + } +} + +/** + * gst_h264_dpb_get_pictures_long_term_ref: + * @dpb: a #GstH264Dpb + * @out: (out): a list of #GstH264Dpb + * + * Retrieve all long-term reference pictures from @dpb + */ +void +gst_h264_dpb_get_pictures_long_term_ref (GstH264Dpb * dpb, GList ** out) +{ + gint i; + + g_return_if_fail (dpb != NULL); + g_return_if_fail (out != NULL); + + for (i = 0; i < dpb->pic_list->len; i++) { + GstH264Picture *picture = + g_array_index (dpb->pic_list, GstH264Picture *, i); + + if (picture->ref && picture->long_term) + *out = g_list_append (*out, gst_h264_picture_ref (picture)); + } +} + +/** + * gst_h264_dpb_get_pictures_all: + * @dpb: a #GstH264Dpb + * + * Return: (transfer full): a #GArray of #GstH264Picture stored in @dpb + */ +GArray * +gst_h264_dpb_get_pictures_all (GstH264Dpb * dpb) +{ + g_return_val_if_fail (dpb != NULL, NULL); + + return g_array_ref (dpb->pic_list); +} + +/** + * gst_h264_dpb_get_size: + * @dpb: a #GstH264Dpb + * + * Return: the length of stored dpb array + */ +gint +gst_h264_dpb_get_size (GstH264Dpb * dpb) +{ + g_return_val_if_fail (dpb != NULL, -1); + + return dpb->pic_list->len; +} + +/** + * gst_h264_dpb_is_full: + * @dpb: a #GstH264Dpb + * + * Return: %TRUE if @dpb is full + */ +gboolean +gst_h264_dpb_is_full (GstH264Dpb * dpb) +{ + g_return_val_if_fail (dpb != NULL, -1); + + return dpb->pic_list->len >= dpb->max_num_pics; +} diff --git a/sys/d3d11/gsth264picture.h b/sys/d3d11/gsth264picture.h new file mode 100644 index 0000000000..bbc6fceb78 --- /dev/null +++ b/sys/d3d11/gsth264picture.h @@ -0,0 +1,216 @@ +/* GStreamer + * Copyright (C) 2019 Seungha Yang + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifndef __GST_H264_PICTURE_H__ +#define __GST_H264_PICTURE_H__ + +#include +#include + +G_BEGIN_DECLS + +#define GST_TYPE_H264_PICTURE (gst_h264_picture_get_type()) +#define GST_IS_H264_PICTURE(obj) (GST_IS_MINI_OBJECT_TYPE(obj, GST_TYPE_H264_PICTURE)) +#define GST_H264_PICTURE(obj) ((GstH264Picture *)obj) +#define GST_H264_PICTURE_CAST(obj) (GST_H264_PICTURE(obj)) + +typedef struct _GstH264Slice GstH264Slice; +typedef struct _GstH264Picture GstH264Picture; + +#define GST_H264_DPB_MAX_SIZE 32 + +struct _GstH264Slice +{ + GstH264SliceHdr header; + + /* parsed nal unit (doesn't take ownership of raw data) */ + GstH264NalUnit nalu; +}; + +typedef enum +{ + GST_H264_PICTURE_FIELD_FRAME, + GST_H264_PICTURE_FILED_TOP_FIELD, + GST_H264_PICTURE_FIELD_BOTTOM_FIELD, +} GstH264PictureField; + +struct _GstH264Picture +{ + GstMiniObject parent; + + GstH264SliceType type; + + GstClockTime pts; + + guint8 pic_order_cnt_type; /* SPS */ + gint32 top_field_order_cnt; + gint32 bottom_field_order_cnt; + + gint pic_order_cnt; + gint pic_order_cnt_msb; + gint pic_order_cnt_lsb; + gint delta_pic_order_cnt_bottom; + gint delta_pic_order_cnt0; + gint delta_pic_order_cnt1; + + gint pic_num; + gint long_term_pic_num; + gint frame_num; + gint frame_num_offset; + gint frame_num_wrap; + gint long_term_frame_idx; + + gint nal_ref_idc; + gboolean idr; + gint idr_pic_id; + gboolean ref; + gboolean long_term; + gboolean outputted; + gboolean mem_mgmt_5; + + gboolean nonexisting; + + GstH264PictureField field; + + GstH264DecRefPicMarking dec_ref_pic_marking; + + gpointer user_data; + GDestroyNotify notify; +}; + +G_GNUC_INTERNAL +GType gst_h264_picture_get_type (void); + +G_GNUC_INTERNAL +GstH264Picture * gst_h264_picture_new (void); + +G_GNUC_INTERNAL +static inline GstH264Picture * +gst_h264_picture_ref (GstH264Picture * picture) +{ + return (GstH264Picture *) gst_mini_object_ref (GST_MINI_OBJECT_CAST (picture)); +} + +G_GNUC_INTERNAL +static inline void +gst_h264_picture_unref (GstH264Picture * picture) +{ + gst_mini_object_unref (GST_MINI_OBJECT_CAST (picture)); +} + +G_GNUC_INTERNAL +static inline gboolean +gst_h264_picture_replace (GstH264Picture ** old_picture, + GstH264Picture * new_picture) +{ + return gst_mini_object_replace ((GstMiniObject **) old_picture, + (GstMiniObject *) new_picture); +} + +G_GNUC_INTERNAL +static inline void +gst_h264_picture_clear (GstH264Picture ** picture) +{ + if (picture && *picture) { + gst_h264_picture_unref (*picture); + *picture = NULL; + } +} + +G_GNUC_INTERNAL +void gst_h264_picture_set_user_data (GstH264Picture * picture, + gpointer user_data, + GDestroyNotify notify); + +G_GNUC_INTERNAL +gpointer gst_h264_picture_get_user_data (GstH264Picture * picture); + +/******************* + * GstH264Dpb * + *******************/ +typedef struct _GstH264Dpb GstH264Dpb; + +G_GNUC_INTERNAL +GstH264Dpb * gst_h264_dpb_new (void); + +G_GNUC_INTERNAL +void gst_h264_dpb_set_max_num_pics (GstH264Dpb * dpb, + gint max_num_pics); + +G_GNUC_INTERNAL +gint gst_h264_dpb_get_max_num_pics (GstH264Dpb * dpb); + +G_GNUC_INTERNAL +void gst_h264_dpb_free (GstH264Dpb * dpb); + +G_GNUC_INTERNAL +void gst_h264_dpb_clear (GstH264Dpb * dpb); + +G_GNUC_INTERNAL +void gst_h264_dpb_add (GstH264Dpb * dpb, + GstH264Picture * picture); + +G_GNUC_INTERNAL +void gst_h264_dpb_delete_unused (GstH264Dpb * dpb); + +G_GNUC_INTERNAL +void gst_h264_dpb_delete_by_poc (GstH264Dpb * dpb, + gint poc); + +G_GNUC_INTERNAL +gint gst_h264_dpb_num_ref_pictures (GstH264Dpb * dpb); + +G_GNUC_INTERNAL +void gst_h264_dpb_mark_all_non_ref (GstH264Dpb * dpb); + +G_GNUC_INTERNAL +GstH264Picture * gst_h264_dpb_get_short_ref_by_pic_num (GstH264Dpb * dpb, + gint pic_num); + +G_GNUC_INTERNAL +GstH264Picture * gst_h264_dpb_get_long_ref_by_pic_num (GstH264Dpb * dpb, + gint pic_num); + +G_GNUC_INTERNAL +GstH264Picture * gst_h264_dpb_get_lowest_frame_num_short_ref (GstH264Dpb * dpb); + +G_GNUC_INTERNAL +void gst_h264_dpb_get_pictures_not_outputted (GstH264Dpb * dpb, + GList ** out); + +G_GNUC_INTERNAL +void gst_h264_dpb_get_pictures_short_term_ref (GstH264Dpb * dpb, + GList ** out); + +G_GNUC_INTERNAL +void gst_h264_dpb_get_pictures_long_term_ref (GstH264Dpb * dpb, + GList ** out); + +G_GNUC_INTERNAL +GArray * gst_h264_dpb_get_pictures_all (GstH264Dpb * dpb); + +G_GNUC_INTERNAL +gint gst_h264_dpb_get_size (GstH264Dpb * dpb); + +G_GNUC_INTERNAL +gboolean gst_h264_dpb_is_full (GstH264Dpb * dpb); + +G_END_DECLS + +#endif /* __GST_H264_PICTURE_H__ */ diff --git a/sys/d3d11/meson.build b/sys/d3d11/meson.build index 1380ebe020..1564e767ce 100644 --- a/sys/d3d11/meson.build +++ b/sys/d3d11/meson.build @@ -17,6 +17,13 @@ d3d11_sources = [ 'gstd3d11overlaycompositor.c', ] +d3d11_dec_sources = [ + 'gsth264picture.c', + 'gsth264decoder.c', + 'gstd3d11decoder.c', + 'gstd3d11h264dec.c', +] + dxgi_headers = [ ['dxgi1_6.h', 6], ['dxgi1_5.h', 5], @@ -91,6 +98,14 @@ endif d3d11_conf.set10('HAVE_D3D11SDKLAYERS_H', have_d3d11sdk_h) d3d11_conf.set10('HAVE_DXGIDEBUG_H', have_dxgidebug_h) +# d3d11 video api uses dxva structure for decoding, and dxva.h needs d3d9 types +if cc.has_header('dxva.h') and cc.has_header('d3d9.h') + d3d11_conf.set('HAVE_DXVA_H', 1) + d3d11_sources += d3d11_dec_sources + extra_c_args += ['-DGST_USE_UNSTABLE_API'] + extra_dep += [gstcodecparsers_dep] +endif + configure_file( output: 'd3d11config.h', configuration: d3d11_conf, diff --git a/sys/d3d11/plugin.c b/sys/d3d11/plugin.c index 8152209506..391d11c57e 100644 --- a/sys/d3d11/plugin.c +++ b/sys/d3d11/plugin.c @@ -29,6 +29,10 @@ #include "gstd3d11download.h" #include "gstd3d11colorconvert.h" #include "gstd3d11videosinkbin.h" +#ifdef HAVE_DXVA_H +#include "gstd3d11utils.h" +#include "gstd3d11h264dec.h" +#endif GST_DEBUG_CATEGORY (gst_d3d11_shader_debug); GST_DEBUG_CATEGORY (gst_d3d11_colorconverter_debug); @@ -41,6 +45,10 @@ GST_DEBUG_CATEGORY (gst_d3d11_overlay_compositor_debug); GST_DEBUG_CATEGORY (gst_d3d11_debug_layer_debug); #endif +#ifdef HAVE_DXVA_H +GST_DEBUG_CATEGORY (gst_d3d11_h264_dec_debug); +#endif + static gboolean plugin_init (GstPlugin * plugin) { @@ -74,6 +82,17 @@ plugin_init (GstPlugin * plugin) gst_element_register (plugin, "d3d11videosink", GST_RANK_SECONDARY - 1, GST_TYPE_D3D11_VIDEO_SINK_BIN); +#ifdef HAVE_DXVA_H + /* DXVA2 API is availble since Windows 8 */ + if (gst_d3d11_is_windows_8_or_greater ()) { + GST_DEBUG_CATEGORY_INIT (gst_d3d11_h264_dec_debug, + "d3d11h264dec", 0, "Direct3D11 H.264 Video Decoder"); + + gst_element_register (plugin, + "d3d11h264dec", GST_RANK_SECONDARY, GST_TYPE_D3D11_H264_DEC); + } +#endif + return TRUE; }