From ca6758350ad0fbb0e1398f84f17adc886167a7e3 Mon Sep 17 00:00:00 2001 From: Thomas Vander Stichele Date: Mon, 17 Dec 2001 18:37:01 +0000 Subject: [PATCH 01/26] building up speed Original commit message from CVS: building up speed From 544c0d75ceb4925c16f59cf2d3ab94d8a7df2374 Mon Sep 17 00:00:00 2001 From: Thijs Vermeir Date: Thu, 16 Oct 2014 10:34:01 +0200 Subject: [PATCH 02/26] rtp: add h265 RTP payloader + depayloader --- gst/rtp/gstrtph265depay.c | 1403 ++++++++++++++++++++++++++++++++++ gst/rtp/gstrtph265depay.h | 98 +++ gst/rtp/gstrtph265pay.c | 1497 +++++++++++++++++++++++++++++++++++++ gst/rtp/gstrtph265pay.h | 82 ++ 4 files changed, 3080 insertions(+) create mode 100644 gst/rtp/gstrtph265depay.c create mode 100644 gst/rtp/gstrtph265depay.h create mode 100644 gst/rtp/gstrtph265pay.c create mode 100644 gst/rtp/gstrtph265pay.h diff --git a/gst/rtp/gstrtph265depay.c b/gst/rtp/gstrtph265depay.c new file mode 100644 index 0000000000..c25bc8150e --- /dev/null +++ b/gst/rtp/gstrtph265depay.c @@ -0,0 +1,1403 @@ +/* GStreamer + * Copyright (C) <2006> Wim Taymans + * Copyright (C) <2014> Jurgen Slowack + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif + +#include +#include + +#include +#include +#include "gstrtph265depay.h" + +GST_DEBUG_CATEGORY_STATIC (rtph265depay_debug); +#define GST_CAT_DEFAULT (rtph265depay_debug) + +/* This is what we'll default to when downstream hasn't + * expressed a restriction or preference via caps */ +#define DEFAULT_BYTE_STREAM TRUE +#define DEFAULT_ACCESS_UNIT FALSE + +/* 3 zero bytes syncword */ +static const guint8 sync_bytes[] = { 0, 0, 0, 1 }; + +static GstStaticPadTemplate gst_rtp_h265_depay_src_template = + GST_STATIC_PAD_TEMPLATE ("src", + GST_PAD_SRC, + GST_PAD_ALWAYS, + GST_STATIC_CAPS ( + /* FIXME - hvc1 and hev1 formats are not supported yet */ + /*"video/x-h265, " + "stream-format = (string) hvc1, alignment = (string) au; " + "video/x-h265, " + "stream-format = (string) hev1, alignment = (string) au; " */ + "video/x-h265, " + "stream-format = (string) byte-stream, alignment = (string) { nal, au }") + ); + +static GstStaticPadTemplate gst_rtp_h265_depay_sink_template = +GST_STATIC_PAD_TEMPLATE ("sink", + GST_PAD_SINK, + GST_PAD_ALWAYS, + GST_STATIC_CAPS ("application/x-rtp, " + "media = (string) \"video\", " + "clock-rate = (int) 90000, " "encoding-name = (string) \"H265\"") + /** optional parameters **/ + /* "profile-space = (int) [ 0, 3 ], " */ + /* "profile-id = (int) [ 0, 31 ], " */ + /* "tier-flag = (int) [ 0, 1 ], " */ + /* "level-id = (int) [ 0, 255 ], " */ + /* "interop-constraints = (string) ANY, " */ + /* "profile-compatibility-indicator = (string) ANY, " */ + /* "sprop-sub-layer-id = (int) [ 0, 6 ], " */ + /* "recv-sub-layer-id = (int) [ 0, 6 ], " */ + /* "max-recv-level-id = (int) [ 0, 255 ], " */ + /* "tx-mode = (string) {MST , SST}, " */ + /* "sprop-vps = (string) ANY, " */ + /* "sprop-sps = (string) ANY, " */ + /* "sprop-pps = (string) ANY, " */ + /* "sprop-sei = (string) ANY, " */ + /* "max-lsr = (int) ANY, " *//* MUST be in the range of MaxLumaSR to 16 * MaxLumaSR, inclusive */ + /* "max-lps = (int) ANY, " *//* MUST be in the range of MaxLumaPS to 16 * MaxLumaPS, inclusive */ + /* "max-cpb = (int) ANY, " *//* MUST be in the range of MaxCPB to 16 * MaxCPB, inclusive */ + /* "max-dpb = (int) [1, 16], " */ + /* "max-br = (int) ANY, " *//* MUST be in the range of MaxBR to 16 * MaxBR, inclusive, for the highest level */ + /* "max-tr = (int) ANY, " *//* MUST be in the range of MaxTileRows to 16 * MaxTileRows, inclusive, for the highest level */ + /* "max-tc = (int) ANY, " *//* MUST be in the range of MaxTileCols to 16 * MaxTileCols, inclusive, for the highest level */ + /* "max-fps = (int) ANY, " */ + /* "sprop-max-don-diff = (int) [0, 32767], " */ + /* "sprop-depack-buf-nalus = (int) [0, 32767], " */ + /* "sprop-depack-buf-nalus = (int) [0, 4294967295], " */ + /* "depack-buf-cap = (int) [1, 4294967295], " */ + /* "sprop-segmentation-id = (int) [0, 3], " */ + /* "sprop-spatial-segmentation-idc = (string) ANY, " */ + /* "dec-parallel-cap = (string) ANY, " */ + ); + +#define gst_rtp_h265_depay_parent_class parent_class +G_DEFINE_TYPE (GstRtpH265Depay, gst_rtp_h265_depay, + GST_TYPE_RTP_BASE_DEPAYLOAD); + +static void gst_rtp_h265_depay_finalize (GObject * object); + +static GstStateChangeReturn gst_rtp_h265_depay_change_state (GstElement * + element, GstStateChange transition); + +static GstBuffer *gst_rtp_h265_depay_process (GstRTPBaseDepayload * depayload, + GstBuffer * buf); +static gboolean gst_rtp_h265_depay_setcaps (GstRTPBaseDepayload * filter, + GstCaps * caps); +static gboolean gst_rtp_h265_depay_handle_event (GstRTPBaseDepayload * depay, + GstEvent * event); + +static void +gst_rtp_h265_depay_class_init (GstRtpH265DepayClass * klass) +{ + GObjectClass *gobject_class; + GstElementClass *gstelement_class; + GstRTPBaseDepayloadClass *gstrtpbasedepayload_class; + + gobject_class = (GObjectClass *) klass; + gstelement_class = (GstElementClass *) klass; + gstrtpbasedepayload_class = (GstRTPBaseDepayloadClass *) klass; + + gobject_class->finalize = gst_rtp_h265_depay_finalize; + + gst_element_class_add_pad_template (gstelement_class, + gst_static_pad_template_get (&gst_rtp_h265_depay_src_template)); + gst_element_class_add_pad_template (gstelement_class, + gst_static_pad_template_get (&gst_rtp_h265_depay_sink_template)); + + gst_element_class_set_static_metadata (gstelement_class, + "RTP H265 depayloader", "Codec/Depayloader/Network/RTP", + "Extracts H265 video from RTP packets (draft-ietf-payload-rtp-h265-03.txt)", + "Jurgen Slowack "); + gstelement_class->change_state = gst_rtp_h265_depay_change_state; + + gstrtpbasedepayload_class->process = gst_rtp_h265_depay_process; + gstrtpbasedepayload_class->set_caps = gst_rtp_h265_depay_setcaps; + gstrtpbasedepayload_class->handle_event = gst_rtp_h265_depay_handle_event; +} + +static void +gst_rtp_h265_depay_init (GstRtpH265Depay * rtph265depay) +{ + rtph265depay->adapter = gst_adapter_new (); + rtph265depay->picture_adapter = gst_adapter_new (); + rtph265depay->byte_stream = DEFAULT_BYTE_STREAM; + rtph265depay->stream_format = (gchar *) g_malloc (10); + rtph265depay->merge = DEFAULT_ACCESS_UNIT; + rtph265depay->vps = g_ptr_array_new_with_free_func ( + (GDestroyNotify) gst_buffer_unref); + rtph265depay->sps = g_ptr_array_new_with_free_func ( + (GDestroyNotify) gst_buffer_unref); + rtph265depay->pps = g_ptr_array_new_with_free_func ( + (GDestroyNotify) gst_buffer_unref); +} + +static void +gst_rtp_h265_depay_reset (GstRtpH265Depay * rtph265depay) +{ + gst_adapter_clear (rtph265depay->adapter); + rtph265depay->wait_start = TRUE; + gst_adapter_clear (rtph265depay->picture_adapter); + rtph265depay->picture_start = FALSE; + rtph265depay->last_keyframe = FALSE; + rtph265depay->last_ts = 0; + rtph265depay->current_fu_type = 0; + rtph265depay->new_codec_data = FALSE; + g_ptr_array_set_size (rtph265depay->vps, 0); + g_ptr_array_set_size (rtph265depay->sps, 0); + g_ptr_array_set_size (rtph265depay->pps, 0); +} + +static void +gst_rtp_h265_depay_finalize (GObject * object) +{ + GstRtpH265Depay *rtph265depay; + + rtph265depay = GST_RTP_H265_DEPAY (object); + + if (rtph265depay->codec_data) + gst_buffer_unref (rtph265depay->codec_data); + + g_free (rtph265depay->stream_format); + + g_object_unref (rtph265depay->adapter); + g_object_unref (rtph265depay->picture_adapter); + + g_ptr_array_free (rtph265depay->vps, TRUE); + g_ptr_array_free (rtph265depay->sps, TRUE); + g_ptr_array_free (rtph265depay->pps, TRUE); + + G_OBJECT_CLASS (parent_class)->finalize (object); +} + +static void +gst_rtp_h265_depay_negotiate (GstRtpH265Depay * rtph265depay) +{ + GstCaps *caps; + gint byte_stream = -1; + gint merge = -1; + + caps = + gst_pad_get_allowed_caps (GST_RTP_BASE_DEPAYLOAD_SRCPAD (rtph265depay)); + + GST_DEBUG_OBJECT (rtph265depay, "allowed caps: %" GST_PTR_FORMAT, caps); + + if (caps) { + if (gst_caps_get_size (caps) > 0) { + GstStructure *s = gst_caps_get_structure (caps, 0); + const gchar *str = NULL; + + if ((str = gst_structure_get_string (s, "stream-format"))) { + + strcpy (rtph265depay->stream_format, str); + + if (strcmp (str, "hev1") == 0) { + byte_stream = FALSE; + } else if (strcmp (str, "hvc1") == 0) { + byte_stream = FALSE; + } else if (strcmp (str, "byte-stream") == 0) { + byte_stream = TRUE; + } else { + GST_DEBUG_OBJECT (rtph265depay, "unknown stream-format: %s", str); + } + } + + if ((str = gst_structure_get_string (s, "alignment"))) { + if (strcmp (str, "au") == 0) { + merge = TRUE; + } else if (strcmp (str, "nal") == 0) { + merge = FALSE; + } else { + GST_DEBUG_OBJECT (rtph265depay, "unknown alignment: %s", str); + } + } + } + gst_caps_unref (caps); + } + + if (byte_stream != -1) { + GST_DEBUG_OBJECT (rtph265depay, "downstream requires byte-stream %d", + byte_stream); + rtph265depay->byte_stream = byte_stream; + } else { + GST_DEBUG_OBJECT (rtph265depay, "defaulting to byte-stream %d", + DEFAULT_BYTE_STREAM); + strcpy (rtph265depay->stream_format, "byte-stream"); + rtph265depay->byte_stream = DEFAULT_BYTE_STREAM; + } + if (merge != -1) { + GST_DEBUG_OBJECT (rtph265depay, "downstream requires merge %d", merge); + rtph265depay->merge = merge; + } else { + GST_DEBUG_OBJECT (rtph265depay, "defaulting to merge %d", + DEFAULT_ACCESS_UNIT); + rtph265depay->merge = DEFAULT_ACCESS_UNIT; + } +} + +/* Stolen from bad/gst/mpegtsdemux/payloader_parsers.c */ +/* variable length Exp-Golomb parsing according to H.265 spec section 9.2*/ +static gboolean +read_golomb (GstBitReader * br, guint32 * value) +{ + guint8 b, leading_zeros = -1; + *value = 1; + + for (b = 0; !b; leading_zeros++) { + if (!gst_bit_reader_get_bits_uint8 (br, &b, 1)) + return FALSE; + *value *= 2; + } + + *value = (*value >> 1) - 1; + if (leading_zeros > 0) { + guint32 tmp = 0; + if (!gst_bit_reader_get_bits_uint32 (br, &tmp, leading_zeros)) + return FALSE; + *value += tmp; + } + + return TRUE; +} + +static gboolean +parse_sps (GstMapInfo * map, guint32 * sps_id) +{ /* To parse seq_parameter_set_id */ + GstBitReader br = GST_BIT_READER_INIT (map->data + 15, + map->size - 15); + + if (map->size < 16) + return FALSE; + + if (!read_golomb (&br, sps_id)) + return FALSE; + + return TRUE; +} + +static gboolean +parse_pps (GstMapInfo * map, guint32 * sps_id, guint32 * pps_id) +{ /* To parse picture_parameter_set_id */ + GstBitReader br = GST_BIT_READER_INIT (map->data + 2, + map->size - 2); + + if (map->size < 3) + return FALSE; + + if (!read_golomb (&br, pps_id)) + return FALSE; + if (!read_golomb (&br, sps_id)) + return FALSE; + + return TRUE; +} + + +static gboolean +gst_rtp_h265_set_src_caps (GstRtpH265Depay * rtph265depay) +{ + gboolean res; + GstCaps *srccaps; + + if (!rtph265depay->byte_stream && + (!rtph265depay->new_codec_data || + rtph265depay->vps->len == 0 || rtph265depay->sps->len == 0 + || rtph265depay->pps->len == 0)) + return TRUE; + + srccaps = gst_caps_new_simple ("video/x-h265", + "stream-format", G_TYPE_STRING, + rtph265depay->stream_format, + "alignment", G_TYPE_STRING, rtph265depay->merge ? "au" : "nal", NULL); + + if (!rtph265depay->byte_stream) { + + GstBuffer *codec_data; + gint i = 0; + gint len; + guint num_vps = rtph265depay->vps->len; + guint num_sps = rtph265depay->sps->len; + guint num_pps = rtph265depay->pps->len; + GstMapInfo map, nalmap; + guint8 *data; + gint nl; + guint8 num_arrays = 0; + guint new_size; + GstBitReader br; + guint32 tmp; + guint8 tmp8 = 0; + guint32 max_sub_layers_minus1, temporal_id_nesting_flag, chroma_format_idc, + bit_depth_luma_minus8, bit_depth_chroma_minus8, + min_spatial_segmentation_idc; + + /* Fixme: Current implementation is not embedding SEI in codec_data */ + + if (num_sps == 0) + return FALSE; + + /* start with 23 bytes header */ + len = 23; + + num_arrays = (num_vps > 0) + (num_sps > 0) + (num_pps > 0); + len += num_arrays; + + /* add size of vps, sps & pps */ + for (i = 0; i < num_vps; i++) + len += 2 + gst_buffer_get_size (g_ptr_array_index (rtph265depay->vps, i)); + for (i = 0; i < num_sps; i++) + len += 2 + gst_buffer_get_size (g_ptr_array_index (rtph265depay->sps, i)); + for (i = 0; i < num_pps; i++) + len += 2 + gst_buffer_get_size (g_ptr_array_index (rtph265depay->pps, i)); + + GST_DEBUG_OBJECT (rtph265depay, + "constructing codec_data: num_vps =%d num_sps=%d, num_pps=%d", num_vps, + num_sps, num_pps); + + codec_data = gst_buffer_new_and_alloc (len); + g_debug ("alloc_len: %u", len); + gst_buffer_map (codec_data, &map, GST_MAP_READWRITE); + data = map.data; + + memset (data, 0, map.size); + + /* Parsing sps to get the info required further on */ + + gst_buffer_map (g_ptr_array_index (rtph265depay->sps, 0), &nalmap, + GST_MAP_READ); + + max_sub_layers_minus1 = ((nalmap.data[2]) >> 1) & 0x07; + temporal_id_nesting_flag = nalmap.data[2] & 0x01; + + gst_bit_reader_init (&br, nalmap.data + 15, nalmap.size - 15); + + read_golomb (&br, &tmp); /* sps_seq_parameter_set_id */ + read_golomb (&br, &chroma_format_idc); /* chroma_format_idc */ + + if (chroma_format_idc == 3) + + gst_bit_reader_get_bits_uint8 (&br, &tmp8, 1); /* separate_colour_plane_flag */ + + read_golomb (&br, &tmp); /* pic_width_in_luma_samples */ + read_golomb (&br, &tmp); /* pic_height_in_luma_samples */ + + gst_bit_reader_get_bits_uint8 (&br, &tmp8, 1); /* conformance_window_flag */ + if (tmp8) { + read_golomb (&br, &tmp); /* conf_win_left_offset */ + read_golomb (&br, &tmp); /* conf_win_right_offset */ + read_golomb (&br, &tmp); /* conf_win_top_offset */ + read_golomb (&br, &tmp); /* conf_win_bottom_offset */ + } + + read_golomb (&br, &bit_depth_luma_minus8); /* bit_depth_luma_minus8 */ + read_golomb (&br, &bit_depth_chroma_minus8); /* bit_depth_chroma_minus8 */ + + GST_DEBUG_OBJECT (rtph265depay, + "Ignoring min_spatial_segmentation for now (assuming zero)"); + + min_spatial_segmentation_idc = 0; /* NOTE - we ignore this for now, but in a perfect world, we should continue parsing to obtain the real value */ + + nl = nalmap.size; + + gst_buffer_unmap (g_ptr_array_index (rtph265depay->sps, 0), &nalmap); + + /* HEVCDecoderConfigurationVersion = 1 */ + data[0] = 1; + + /* Copy from profile_tier_level (Rec. ITU-T H.265 (04/2013) section 7.3.3 + * + * profile_space | tier_flat | profile_idc | + * profile_compatibility_flags | constraint_indicator_flags | + * level_idc | progressive_source_flag | interlaced_source_flag + * non_packed_constraint_flag | frame_only_constraint_flag + * reserved_zero_44bits | level_idc */ + gst_buffer_map (g_ptr_array_index (rtph265depay->sps, 0), &nalmap, + GST_MAP_READ); + for (i = 0; i < 12; i++) + data[i + 1] = nalmap.data[i]; + gst_buffer_unmap (g_ptr_array_index (rtph265depay->sps, 0), &nalmap); + + /* min_spatial_segmentation_idc */ + GST_WRITE_UINT16_BE (data + 13, min_spatial_segmentation_idc); + data[13] |= 0xf0; + data[15] = 0xfc; /* keeping parrallelismType as zero (unknown) */ + data[16] = 0xfc | chroma_format_idc; + data[17] = 0xf8 | bit_depth_luma_minus8; + data[18] = 0xf8 | bit_depth_chroma_minus8; + data[19] = 0x00; /* keep avgFrameRate as unspecified */ + data[20] = 0x00; /* keep avgFrameRate as unspecified */ + /* constFrameRate(2 bits): 0, stream may or may not be of constant framerate + * numTemporalLayers (3 bits): number of temporal layers, value from SPS + * TemporalIdNested (1 bit): sps_temporal_id_nesting_flag from SPS + * lengthSizeMinusOne (2 bits): plus 1 indicates the length of the NALUnitLength */ + data[21] = + 0x00 | ((max_sub_layers_minus1 + + 1) << 3) | (temporal_id_nesting_flag << 2) | (nl - 1); + GST_WRITE_UINT8 (data + 22, num_arrays); /* numOfArrays */ + + data += 23; + + /* copy all VPS */ + if (num_vps > 0) { + /* array_completeness | reserved_zero bit | nal_unit_type */ + data[0] = 0x00 | 0x20; + data++; + + GST_WRITE_UINT16_BE (data, num_vps); + data += 2; + + for (i = 0; i < num_vps; i++) { + gsize nal_size = + gst_buffer_get_size (g_ptr_array_index (rtph265depay->vps, i)); + GST_WRITE_UINT16_BE (data, nal_size); + gst_buffer_extract (g_ptr_array_index (rtph265depay->vps, i), 0, + data + 2, nal_size); + data += 2 + nal_size; + GST_DEBUG_OBJECT (rtph265depay, "Copied VPS %d of length %u", i, + (guint) nal_size); + } + } + + /* copy all SPS */ + if (num_sps > 0) { + /* array_completeness | reserved_zero bit | nal_unit_type */ + data[0] = 0x00 | 0x21; + data++; + + GST_WRITE_UINT16_BE (data, num_sps); + data += 2; + + for (i = 0; i < num_sps; i++) { + gsize nal_size = + gst_buffer_get_size (g_ptr_array_index (rtph265depay->sps, i)); + GST_WRITE_UINT16_BE (data, nal_size); + gst_buffer_extract (g_ptr_array_index (rtph265depay->sps, i), 0, + data + 2, nal_size); + data += 2 + nal_size; + GST_DEBUG_OBJECT (rtph265depay, "Copied SPS %d of length %u", i, + (guint) nal_size); + } + } + + /* copy all PPS */ + if (num_pps > 0) { + /* array_completeness | reserved_zero bit | nal_unit_type */ + data[0] = 0x00 | 0x22; + data++; + + GST_WRITE_UINT16_BE (data, num_pps); + data += 2; + + for (i = 0; i < num_pps; i++) { + gsize nal_size = + gst_buffer_get_size (g_ptr_array_index (rtph265depay->pps, i)); + GST_WRITE_UINT16_BE (data, nal_size); + gst_buffer_extract (g_ptr_array_index (rtph265depay->pps, i), 0, + data + 2, nal_size); + data += 2 + nal_size; + GST_DEBUG_OBJECT (rtph265depay, "Copied PPS %d of length %u", i, + (guint) nal_size); + } + } + + new_size = data - map.data; + gst_buffer_unmap (codec_data, &map); + gst_buffer_set_size (codec_data, new_size); + + gst_caps_set_simple (srccaps, + "codec_data", GST_TYPE_BUFFER, codec_data, NULL); + gst_buffer_unref (codec_data); + } + + res = gst_pad_set_caps (GST_RTP_BASE_DEPAYLOAD_SRCPAD (rtph265depay), + srccaps); + gst_caps_unref (srccaps); + + if (res) + rtph265depay->new_codec_data = FALSE; + + return res; +} + +gboolean +gst_rtp_h265_add_vps_sps_pps (GstElement * rtph265, GPtrArray * vps_array, + GPtrArray * sps_array, GPtrArray * pps_array, GstBuffer * nal) +{ + GstMapInfo map; + guchar type; + guint i; + + gst_buffer_map (nal, &map, GST_MAP_READ); + + type = (map.data[0] >> 1) & 0x3f; + + if (type == GST_H265_VPS_NUT) { + guint32 vps_id = (map.data[2] >> 4) & 0x0f; + + for (i = 0; i < vps_array->len; i++) { + GstBuffer *vps = g_ptr_array_index (vps_array, i); + GstMapInfo vpsmap; + guint32 tmp_vps_id; + + gst_buffer_map (vps, &vpsmap, GST_MAP_READ); + tmp_vps_id = (vpsmap.data[2] >> 4) & 0x0f; + + if (vps_id == tmp_vps_id) { + if (map.size == vpsmap.size && + memcmp (map.data, vpsmap.data, vpsmap.size) == 0) { + GST_LOG_OBJECT (rtph265, "Unchanged VPS %u, not updating", vps_id); + gst_buffer_unmap (vps, &vpsmap); + goto drop; + } else { + gst_buffer_unmap (vps, &vpsmap); + g_ptr_array_remove_index_fast (vps_array, i); + g_ptr_array_add (vps_array, nal); + GST_LOG_OBJECT (rtph265, "Modified VPS %u, replacing", vps_id); + goto done; + } + } + gst_buffer_unmap (vps, &vpsmap); + } + GST_LOG_OBJECT (rtph265, "Adding new VPS %u", vps_id); + g_ptr_array_add (vps_array, nal); + } else if (type == GST_H265_SPS_NUT) { + guint32 sps_id; + + if (!parse_sps (&map, &sps_id)) { + GST_WARNING_OBJECT (rtph265, "Invalid SPS," + " can't parse seq_parameter_set_id"); + goto drop; + } + + for (i = 0; i < sps_array->len; i++) { + GstBuffer *sps = g_ptr_array_index (sps_array, i); + GstMapInfo spsmap; + guint32 tmp_sps_id; + + gst_buffer_map (sps, &spsmap, GST_MAP_READ); + parse_sps (&spsmap, &tmp_sps_id); + + if (sps_id == tmp_sps_id) { + if (map.size == spsmap.size && + memcmp (map.data, spsmap.data, spsmap.size) == 0) { + GST_LOG_OBJECT (rtph265, "Unchanged SPS %u, not updating", sps_id); + gst_buffer_unmap (sps, &spsmap); + goto drop; + } else { + gst_buffer_unmap (sps, &spsmap); + g_ptr_array_remove_index_fast (sps_array, i); + g_ptr_array_add (sps_array, nal); + GST_LOG_OBJECT (rtph265, "Modified SPS %u, replacing", sps_id); + goto done; + } + } + gst_buffer_unmap (sps, &spsmap); + } + GST_LOG_OBJECT (rtph265, "Adding new SPS %u", sps_id); + g_ptr_array_add (sps_array, nal); + } else if (type == GST_H265_PPS_NUT) { + guint32 sps_id; + guint32 pps_id; + + if (!parse_pps (&map, &sps_id, &pps_id)) { + GST_WARNING_OBJECT (rtph265, "Invalid PPS," + " can't parse seq_parameter_set_id or pic_parameter_set_id"); + goto drop; + } + + for (i = 0; i < pps_array->len; i++) { + GstBuffer *pps = g_ptr_array_index (pps_array, i); + GstMapInfo ppsmap; + guint32 tmp_sps_id; + guint32 tmp_pps_id; + + + gst_buffer_map (pps, &ppsmap, GST_MAP_READ); + parse_pps (&ppsmap, &tmp_sps_id, &tmp_pps_id); + + if (sps_id == tmp_sps_id && pps_id == tmp_pps_id) { + if (map.size == ppsmap.size && + memcmp (map.data, ppsmap.data, ppsmap.size) == 0) { + GST_LOG_OBJECT (rtph265, "Unchanged PPS %u:%u, not updating", sps_id, + pps_id); + gst_buffer_unmap (pps, &ppsmap); + goto drop; + } else { + gst_buffer_unmap (pps, &ppsmap); + g_ptr_array_remove_index_fast (pps_array, i); + g_ptr_array_add (pps_array, nal); + GST_LOG_OBJECT (rtph265, "Modified PPS %u:%u, replacing", + sps_id, pps_id); + goto done; + } + } + gst_buffer_unmap (pps, &ppsmap); + } + GST_LOG_OBJECT (rtph265, "Adding new PPS %u:%i", sps_id, pps_id); + g_ptr_array_add (pps_array, nal); + } else { + goto drop; + } + +done: + gst_buffer_unmap (nal, &map); + + return TRUE; + +drop: + gst_buffer_unmap (nal, &map); + gst_buffer_unref (nal); + + return FALSE; +} + + +static void +gst_rtp_h265_depay_add_vps_sps_pps (GstRtpH265Depay * rtph265depay, + GstBuffer * nal) +{ + if (gst_rtp_h265_add_vps_sps_pps (GST_ELEMENT (rtph265depay), + rtph265depay->vps, rtph265depay->sps, rtph265depay->pps, nal)) + rtph265depay->new_codec_data = TRUE; +} + +static gboolean +gst_rtp_h265_depay_setcaps (GstRTPBaseDepayload * depayload, GstCaps * caps) +{ + gint clock_rate; + GstStructure *structure = gst_caps_get_structure (caps, 0); + GstRtpH265Depay *rtph265depay; + const gchar *ps; + GstBuffer *codec_data; + GstMapInfo map; + guint8 *ptr; + + rtph265depay = GST_RTP_H265_DEPAY (depayload); + + if (!gst_structure_get_int (structure, "clock-rate", &clock_rate)) + clock_rate = 90000; + depayload->clock_rate = clock_rate; + + /* Base64 encoded, comma separated config NALs */ + ps = gst_structure_get_string (structure, "sprop-parameter-sets"); + + /* negotiate with downstream w.r.t. output format and alignment */ + gst_rtp_h265_depay_negotiate (rtph265depay); + + if (rtph265depay->byte_stream && ps != NULL) { + /* for bytestream we only need the parameter sets but we don't error out + * when they are not there, we assume they are in the stream. */ + gchar **params; + guint len, total; + gint i; + + params = g_strsplit (ps, ",", 0); + + /* count total number of bytes in base64. Also include the sync bytes in + * front of the params. */ + len = 0; + for (i = 0; params[i]; i++) { + len += strlen (params[i]); + len += sizeof (sync_bytes); + } + /* we seriously overshoot the length, but it's fine. */ + codec_data = gst_buffer_new_and_alloc (len); + + gst_buffer_map (codec_data, &map, GST_MAP_WRITE); + ptr = map.data; + total = 0; + for (i = 0; params[i]; i++) { + guint save = 0; + gint state = 0; + + GST_DEBUG_OBJECT (depayload, "decoding param %d (%s)", i, params[i]); + memcpy (ptr, sync_bytes, sizeof (sync_bytes)); + ptr += sizeof (sync_bytes); + len = + g_base64_decode_step (params[i], strlen (params[i]), ptr, &state, + &save); + GST_DEBUG_OBJECT (depayload, "decoded %d bytes", len); + total += len + sizeof (sync_bytes); + ptr += len; + } + gst_buffer_unmap (codec_data, &map); + gst_buffer_resize (codec_data, 0, total); + g_strfreev (params); + + /* keep the codec_data, we need to send it as the first buffer. We cannot + * push it in the adapter because the adapter might be flushed on discont. + */ + if (rtph265depay->codec_data) + gst_buffer_unref (rtph265depay->codec_data); + rtph265depay->codec_data = codec_data; + } else if (!rtph265depay->byte_stream) { + gchar **params; + gint i; + + if (ps == NULL) + goto incomplete_caps; + + params = g_strsplit (ps, ",", 0); + + GST_DEBUG_OBJECT (depayload, "we have %d params", g_strv_length (params)); + + /* start with 23 bytes header */ + for (i = 0; params[i]; i++) { + GstBuffer *nal; + GstMapInfo nalmap; + gsize nal_len; + guint save = 0; + gint state = 0; + + nal_len = strlen (params[i]); + nal = gst_buffer_new_and_alloc (nal_len); + gst_buffer_map (nal, &nalmap, GST_MAP_READWRITE); + + nal_len = + g_base64_decode_step (params[i], nal_len, nalmap.data, &state, &save); + + GST_DEBUG_OBJECT (depayload, "adding param %d as %s", i, + (((nalmap.data[0] >> 1) & 0x3f) == + 32) ? "VPS" : (((nalmap.data[0] >> 1) & 0x3f) == + 33) ? "SPS" : "PPS"); + + gst_buffer_unmap (nal, &nalmap); + gst_buffer_set_size (nal, nal_len); + + gst_rtp_h265_depay_add_vps_sps_pps (rtph265depay, nal); + } + g_strfreev (params); + + if (rtph265depay->sps->len == 0 || rtph265depay->pps->len == 0) + goto incomplete_caps; + } + + return gst_rtp_h265_set_src_caps (rtph265depay); + + /* ERRORS */ +incomplete_caps: + { + GST_DEBUG_OBJECT (depayload, "we have incomplete caps," + " doing setcaps later"); + return TRUE; + } +} + +static GstBuffer * +gst_rtp_h265_complete_au (GstRtpH265Depay * rtph265depay, + GstClockTime * out_timestamp, gboolean * out_keyframe) +{ + guint outsize; + GstBuffer *outbuf; + + /* we had a picture in the adapter and we completed it */ + GST_DEBUG_OBJECT (rtph265depay, "taking completed AU"); + outsize = gst_adapter_available (rtph265depay->picture_adapter); + outbuf = gst_adapter_take_buffer (rtph265depay->picture_adapter, outsize); + + *out_timestamp = rtph265depay->last_ts; + *out_keyframe = rtph265depay->last_keyframe; + + rtph265depay->last_keyframe = FALSE; + rtph265depay->picture_start = FALSE; + + return outbuf; +} + +/* VPS/SPS/PPS/RADL/TSA/RASL/IDR/CRA is considered key, all others DELTA; + * so downstream waiting for keyframe can pick up at VPS/SPS/PPS/IDR */ + +#define NAL_TYPE_IS_PARAMETER_SET(nt) ( ((nt) == GST_H265_VPS_NUT)\ + || ((nt) == GST_H265_SPS_NUT)\ + || ((nt) == GST_H265_PPS_NUT) ) + +#define NAL_TYPE_IS_CODED_SLICE_SEGMENT(nt) ( ((nt) == GST_H265_NAL_SLICE_TRAIL_N)\ + || ((nt) == GST_H265_NAL_SLICE_TRAIL_R)\ + || ((nt) == GST_H265_NAL_SLICE_TSA_N)\ + || ((nt) == GST_H265_NAL_SLICE_TSA_R)\ + || ((nt) == GST_H265_NAL_SLICE_STSA_N)\ + || ((nt) == GST_H265_NAL_SLICE_STSA_R)\ + || ((nt) == GST_H265_NAL_SLICE_RASL_N)\ + || ((nt) == GST_H265_NAL_SLICE_RASL_R)\ + || ((nt) == GST_H265_NAL_SLICE_BLA_W_LP)\ + || ((nt) == GST_H265_NAL_SLICE_BLA_W_RADL)\ + || ((nt) == GST_H265_NAL_SLICE_BLA_N_LP)\ + || ((nt) == GST_H265_NAL_SLICE_IDR_W_RADL)\ + || ((nt) == GST_H265_NAL_SLICE_IDR_N_LP)\ + || ((nt) == GST_H265_NAL_SLICE_CRA_NUT) ) + +#define NAL_TYPE_IS_KEY(nt) (NAL_TYPE_IS_PARAMETER_SET(nt) || NAL_TYPE_IS_CODED_SLICE_SEGMENT(nt)) + +static GstBuffer * +gst_rtp_h265_depay_handle_nal (GstRtpH265Depay * rtph265depay, GstBuffer * nal, + GstClockTime in_timestamp, gboolean marker) +{ + GstRTPBaseDepayload *depayload = GST_RTP_BASE_DEPAYLOAD (rtph265depay); + gint nal_type; + GstMapInfo map; + GstBuffer *outbuf = NULL; + GstClockTime out_timestamp; + gboolean keyframe, out_keyframe; + + gst_buffer_map (nal, &map, GST_MAP_READ); + if (G_UNLIKELY (map.size < 5)) + goto short_nal; + + nal_type = (map.data[4] >> 1) & 0x3f; + GST_DEBUG_OBJECT (rtph265depay, "handle NAL type %d (RTP marker bit %d)", + nal_type, marker); + + keyframe = NAL_TYPE_IS_KEY (nal_type); + + out_keyframe = keyframe; + out_timestamp = in_timestamp; + + if (!rtph265depay->byte_stream) { + if (NAL_TYPE_IS_PARAMETER_SET (nal_type)) { + gst_rtp_h265_depay_add_vps_sps_pps (rtph265depay, + gst_buffer_copy_region (nal, GST_BUFFER_COPY_ALL, + 4, gst_buffer_get_size (nal) - 4)); + gst_buffer_unmap (nal, &map); + gst_buffer_unref (nal); + return NULL; + } else if (rtph265depay->sps->len == 0 || rtph265depay->pps->len == 0) { + /* Down push down any buffer in non-bytestream mode if the SPS/PPS haven't + * go through yet + */ + gst_pad_push_event (GST_RTP_BASE_DEPAYLOAD_SINKPAD (depayload), + gst_event_new_custom (GST_EVENT_CUSTOM_UPSTREAM, + gst_structure_new ("GstForceKeyUnit", + "all-headers", G_TYPE_BOOLEAN, TRUE, NULL))); + gst_buffer_unmap (nal, &map); + gst_buffer_unref (nal); + return NULL; + } + + if (rtph265depay->new_codec_data && + rtph265depay->sps->len > 0 && rtph265depay->pps->len > 0) + gst_rtp_h265_set_src_caps (rtph265depay); + } + + if (rtph265depay->merge) { + gboolean start = FALSE, complete = FALSE; + + /* marker bit isn't mandatory so in the following code we try to detect + * an AU boundary (see H.265 spec section 7.4.2.4.4) */ + if (!marker) { + + if (NAL_TYPE_IS_CODED_SLICE_SEGMENT (nal_type)) { + /* A NAL unit (X) ends an access unit if the next-occurring VCL NAL unit (Y) has the high-order bit of the first byte after its NAL unit header equal to 1 */ + start = TRUE; + if (((map.data[6] >> 7) & 0x01) == 1) { + complete = TRUE; + } + complete = TRUE; + } else if ((nal_type >= 32 && nal_type <= 35) + || nal_type >= 39 || (nal_type >= 41 && nal_type <= 44) + || (nal_type >= 48 && nal_type <= 55)) { + /* VPS, SPS, PPS, SEI, ... terminate an access unit */ + complete = TRUE; + } + GST_DEBUG_OBJECT (depayload, "start %d, complete %d", start, complete); + + if (complete && rtph265depay->picture_start) + outbuf = gst_rtp_h265_complete_au (rtph265depay, &out_timestamp, + &out_keyframe); + } + /* add to adapter */ + gst_buffer_unmap (nal, &map); + + GST_DEBUG_OBJECT (depayload, "adding NAL to picture adapter"); + gst_adapter_push (rtph265depay->picture_adapter, nal); + rtph265depay->last_ts = in_timestamp; + rtph265depay->last_keyframe |= keyframe; + rtph265depay->picture_start |= start; + + if (marker) + outbuf = gst_rtp_h265_complete_au (rtph265depay, &out_timestamp, + &out_keyframe); + } else { + /* no merge, output is input nal */ + GST_DEBUG_OBJECT (depayload, "using NAL as output"); + outbuf = nal; + gst_buffer_unmap (nal, &map); + } + + if (outbuf) { + /* prepend codec_data */ + if (rtph265depay->codec_data) { + GST_DEBUG_OBJECT (depayload, "prepending codec_data"); + outbuf = gst_buffer_append (rtph265depay->codec_data, outbuf); + rtph265depay->codec_data = NULL; + out_keyframe = TRUE; + } + outbuf = gst_buffer_make_writable (outbuf); + + GST_BUFFER_TIMESTAMP (outbuf) = out_timestamp; + + if (out_keyframe) + GST_BUFFER_FLAG_UNSET (outbuf, GST_BUFFER_FLAG_DELTA_UNIT); + else + GST_BUFFER_FLAG_SET (outbuf, GST_BUFFER_FLAG_DELTA_UNIT); + } + + return outbuf; + + /* ERRORS */ +short_nal: + { + GST_WARNING_OBJECT (depayload, "dropping short NAL"); + gst_buffer_unmap (nal, &map); + gst_buffer_unref (nal); + return NULL; + } +} + +static GstBuffer * +gst_rtp_h265_push_fragmentation_unit (GstRtpH265Depay * rtph265depay, + gboolean send) +{ + guint outsize; + GstMapInfo map; + GstBuffer *outbuf; + + outsize = gst_adapter_available (rtph265depay->adapter); + outbuf = gst_adapter_take_buffer (rtph265depay->adapter, outsize); + + gst_buffer_map (outbuf, &map, GST_MAP_WRITE); + GST_DEBUG_OBJECT (rtph265depay, "output %d bytes", outsize); + + if (rtph265depay->byte_stream) { + memcpy (map.data, sync_bytes, sizeof (sync_bytes)); + } else { + goto not_implemented; + } + gst_buffer_unmap (outbuf, &map); + + rtph265depay->current_fu_type = 0; + + outbuf = gst_rtp_h265_depay_handle_nal (rtph265depay, outbuf, + rtph265depay->fu_timestamp, rtph265depay->fu_marker); + + if (send && outbuf) { + gst_rtp_base_depayload_push (GST_RTP_BASE_DEPAYLOAD (rtph265depay), outbuf); + outbuf = NULL; + } + return outbuf; + +not_implemented: + { + GST_ERROR_OBJECT (rtph265depay, + ("Only bytestream format is currently supported.")); + gst_buffer_unmap (outbuf, &map); + return NULL; + } +} + +static GstBuffer * +gst_rtp_h265_depay_process (GstRTPBaseDepayload * depayload, GstBuffer * buf) +{ + GstRtpH265Depay *rtph265depay; + GstBuffer *outbuf = NULL; + guint8 nal_unit_type; + GstRTPBuffer rtp = { NULL }; + + rtph265depay = GST_RTP_H265_DEPAY (depayload); + + /* flush remaining data on discont */ + if (GST_BUFFER_IS_DISCONT (buf)) { + gst_adapter_clear (rtph265depay->adapter); + rtph265depay->wait_start = TRUE; + rtph265depay->current_fu_type = 0; + } + + { + gint payload_len; + guint8 *payload; + guint header_len; + GstMapInfo map; + guint outsize, nalu_size; + GstClockTime timestamp; + gboolean marker; + gboolean donl_present = FALSE; + guint8 nuh_layer_id, nuh_temporal_id_plus1; + guint8 S, E; + guint16 nal_header; + + timestamp = GST_BUFFER_TIMESTAMP (buf); + + gst_rtp_buffer_map (buf, GST_MAP_READ, &rtp); + + payload_len = gst_rtp_buffer_get_payload_len (&rtp); + payload = gst_rtp_buffer_get_payload (&rtp); + marker = gst_rtp_buffer_get_marker (&rtp); + + GST_DEBUG_OBJECT (rtph265depay, "receiving %d bytes", payload_len); + + if (payload_len == 0) + goto empty_packet; + + /* +---------------+---------------+ + * |0|1|2|3|4|5|6|7|0|1|2|3|4|5|6|7| + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * |F| Type | LayerId | TID | + * +-------------+-----------------+ + * + * F must be 0. + * + */ + nal_unit_type = (payload[0] >> 1) & 0x3f; + nuh_layer_id = ((payload[0] & 0x01) << 5) | (payload[1] >> 3); /* should be zero for now but this could change in future HEVC extensions */ + nuh_temporal_id_plus1 = payload[1] & 0x03; + + /* At least two byte header with type */ + header_len = 2; + + GST_DEBUG_OBJECT (rtph265depay, + "NAL header nal_unit_type %d, nuh_temporal_id_plus1 %d", nal_unit_type, + nuh_temporal_id_plus1); + + GST_FIXME_OBJECT (rtph265depay, "Assuming DONL field is not present"); + + /* FIXME - assuming DONL field is not present for now */ + /*donl_present = (tx-mode == "MST") || (sprop-max-don-diff > 0); */ + + /* If FU unit was being processed, but the current nal is of a different + * type. Assume that the remote payloader is buggy (didn't set the end bit + * when the FU ended) and send out what we gathered thusfar */ + if (G_UNLIKELY (rtph265depay->current_fu_type != 0 && + nal_unit_type != rtph265depay->current_fu_type)) + gst_rtp_h265_push_fragmentation_unit (rtph265depay, TRUE); + + switch (nal_unit_type) { + case 48: + { + GST_DEBUG_OBJECT (rtph265depay, "Processing aggregation packet"); + + /* Aggregation packet (section 4.7) */ + + /* An example of an AP packet containing two aggregation units + without the DONL and DOND fields + + 0 1 2 3 + 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + | RTP Header | + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + | PayloadHdr (Type=48) | NALU 1 Size | + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + | NALU 1 HDR | | + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ NALU 1 Data | + | . . . | + | | + + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + | . . . | NALU 2 Size | NALU 2 HDR | + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + | NALU 2 HDR | | + +-+-+-+-+-+-+-+-+ NALU 2 Data | + | . . . | + | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + | :...OPTIONAL RTP padding | + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + */ + + /* strip headers */ + payload += header_len; + payload_len -= header_len; + + rtph265depay->wait_start = FALSE; + + if (donl_present) + goto not_implemented_donl_present; + + while (payload_len > 2) { + + nalu_size = (payload[0] << 8) | payload[1]; + + /* dont include nalu_size */ + if (nalu_size > (payload_len - 2)) + nalu_size = payload_len - 2; + + outsize = nalu_size + sizeof (sync_bytes); + outbuf = gst_buffer_new_and_alloc (outsize); + + gst_buffer_map (outbuf, &map, GST_MAP_WRITE); + if (rtph265depay->byte_stream) { + memcpy (map.data, sync_bytes, sizeof (sync_bytes)); + } else { + goto not_implemented; + } + + /* strip NALU size */ + payload += 2; + payload_len -= 2; + + memcpy (map.data + sizeof (sync_bytes), payload, nalu_size); + gst_buffer_unmap (outbuf, &map); + + gst_adapter_push (rtph265depay->adapter, outbuf); + + payload += nalu_size; + payload_len -= nalu_size; + } + + outsize = gst_adapter_available (rtph265depay->adapter); + outbuf = gst_adapter_take_buffer (rtph265depay->adapter, outsize); + + outbuf = gst_rtp_h265_depay_handle_nal (rtph265depay, outbuf, timestamp, + marker); + break; + } + case 49: + { + GST_DEBUG_OBJECT (rtph265depay, "Processing Fragmentation Unit"); + + /* Fragmentation units (FUs) Section 4.8 */ + + /* The structure of a Fragmentation Unit (FU) + * + * 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + | PayloadHdr (Type=49) | FU header | DONL (cond) | + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-| + | DONL (cond) | | + |-+-+-+-+-+-+-+-+ | + | FU payload | + | | + | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + | :...OPTIONAL RTP padding | + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * + * + */ + + /* strip headers */ + payload += header_len; + payload_len -= header_len; + + /* processing FU header */ + S = (payload[0] & 0x80) == 0x80; + E = (payload[0] & 0x40) == 0x40; + + GST_DEBUG_OBJECT (rtph265depay, + "FU header with S %d, E %d, nal_unit_type %d", S, E, + payload[0] & 0x3f); + + if (rtph265depay->wait_start && !S) + goto waiting_start; + + if (donl_present) + goto not_implemented_donl_present; + + if (S) { + + GST_DEBUG_OBJECT (rtph265depay, "Start of Fragmentation Unit"); + + /* If a new FU unit started, while still processing an older one. + * Assume that the remote payloader is buggy (doesn't set the end + * bit) and send out what we've gathered thusfar */ + if (G_UNLIKELY (rtph265depay->current_fu_type != 0)) + gst_rtp_h265_push_fragmentation_unit (rtph265depay, TRUE); + + rtph265depay->current_fu_type = nal_unit_type; + rtph265depay->fu_timestamp = timestamp; + + rtph265depay->wait_start = FALSE; + + /* reconstruct NAL header */ + nal_header = + ((payload[0] & 0x3f) << 9) | (nuh_layer_id << 3) | + nuh_temporal_id_plus1; + + /* go back one byte so we can copy the payload + two bytes more in the front which + * will be overwritten by the nal_header + */ + payload -= 1; + payload_len += 1; + + nalu_size = payload_len; + outsize = nalu_size + sizeof (sync_bytes); + outbuf = gst_buffer_new_and_alloc (outsize); + + gst_buffer_map (outbuf, &map, GST_MAP_WRITE); + memcpy (map.data + sizeof (sync_bytes), payload, nalu_size); + map.data[sizeof (sync_bytes)] = nal_header >> 8; + map.data[sizeof (sync_bytes) + 1] = nal_header & 0xff; + gst_buffer_unmap (outbuf, &map); + + GST_DEBUG_OBJECT (rtph265depay, "queueing %d bytes", outsize); + + /* and assemble in the adapter */ + gst_adapter_push (rtph265depay->adapter, outbuf); + } else { + + GST_DEBUG_OBJECT (rtph265depay, + "Following part of Fragmentation Unit"); + + /* strip off FU header byte */ + payload += 1; + payload_len -= 1; + + outsize = payload_len; + outbuf = gst_buffer_new_and_alloc (outsize); + gst_buffer_fill (outbuf, 0, payload, outsize); + + GST_DEBUG_OBJECT (rtph265depay, "queueing %d bytes", outsize); + + /* and assemble in the adapter */ + gst_adapter_push (rtph265depay->adapter, outbuf); + } + + outbuf = NULL; + rtph265depay->fu_marker = marker; + + /* if NAL unit ends, flush the adapter */ + if (E) { + outbuf = gst_rtp_h265_push_fragmentation_unit (rtph265depay, FALSE); + GST_DEBUG_OBJECT (rtph265depay, "End of Fragmentation Unit"); + } + break; + } + case 50: + goto not_implemented; /* PACI packets Section 4.9 */ + default: + { + rtph265depay->wait_start = FALSE; + + /* All other cases: Single NAL unit packet Section 4.6 */ + /* the entire payload is the output buffer */ + + if (donl_present) + goto not_implemented_donl_present; + + nalu_size = payload_len; + outsize = nalu_size + sizeof (sync_bytes); + outbuf = gst_buffer_new_and_alloc (outsize); + + gst_buffer_map (outbuf, &map, GST_MAP_WRITE); + if (rtph265depay->byte_stream) { + memcpy (map.data, sync_bytes, sizeof (sync_bytes)); + } else { + goto not_implemented; + } + memcpy (map.data + sizeof (sync_bytes), payload, nalu_size); + gst_buffer_unmap (outbuf, &map); + + outbuf = gst_rtp_h265_depay_handle_nal (rtph265depay, outbuf, timestamp, + marker); + break; + } + } + gst_rtp_buffer_unmap (&rtp); + } + + return outbuf; + + /* ERRORS */ +empty_packet: + { + GST_DEBUG_OBJECT (rtph265depay, "empty packet"); + gst_rtp_buffer_unmap (&rtp); + return NULL; + } +waiting_start: + { + GST_DEBUG_OBJECT (rtph265depay, "waiting for start"); + gst_rtp_buffer_unmap (&rtp); + return NULL; + } +not_implemented_donl_present: + { + GST_ELEMENT_ERROR (rtph265depay, STREAM, FORMAT, + (NULL), ("DONL field present not supported yet")); + gst_rtp_buffer_unmap (&rtp); + return NULL; + } +not_implemented: + { + GST_ELEMENT_ERROR (rtph265depay, STREAM, FORMAT, + (NULL), ("NAL unit type %d not supported yet", nal_unit_type)); + gst_rtp_buffer_unmap (&rtp); + return NULL; + } +} + +static gboolean +gst_rtp_h265_depay_handle_event (GstRTPBaseDepayload * depay, GstEvent * event) +{ + GstRtpH265Depay *rtph265depay; + + rtph265depay = GST_RTP_H265_DEPAY (depay); + + switch (GST_EVENT_TYPE (event)) { + case GST_EVENT_FLUSH_STOP: + gst_rtp_h265_depay_reset (rtph265depay); + break; + default: + break; + } + + return + GST_RTP_BASE_DEPAYLOAD_CLASS (parent_class)->handle_event (depay, event); +} + +static GstStateChangeReturn +gst_rtp_h265_depay_change_state (GstElement * element, + GstStateChange transition) +{ + GstRtpH265Depay *rtph265depay; + GstStateChangeReturn ret; + + rtph265depay = GST_RTP_H265_DEPAY (element); + + switch (transition) { + case GST_STATE_CHANGE_NULL_TO_READY: + break; + case GST_STATE_CHANGE_READY_TO_PAUSED: + gst_rtp_h265_depay_reset (rtph265depay); + break; + default: + break; + } + + ret = GST_ELEMENT_CLASS (parent_class)->change_state (element, transition); + + switch (transition) { + case GST_STATE_CHANGE_READY_TO_NULL: + break; + default: + break; + } + return ret; +} + +gboolean +gst_rtp_h265_depay_plugin_init (GstPlugin * plugin) +{ + GST_DEBUG_CATEGORY_INIT (rtph265depay_debug, "rtph265depay", 0, + "H265 Video RTP Depayloader"); + + return gst_element_register (plugin, "rtph265depay", + GST_RANK_SECONDARY, GST_TYPE_RTP_H265_DEPAY); +} diff --git a/gst/rtp/gstrtph265depay.h b/gst/rtp/gstrtph265depay.h new file mode 100644 index 0000000000..ed49329a3d --- /dev/null +++ b/gst/rtp/gstrtph265depay.h @@ -0,0 +1,98 @@ +/* GStreamer + * Copyright (C) <2006> Wim Taymans + * Copyright (C) <2014> Jurgen Slowack + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifndef __GST_RTP_H265_DEPAY_H__ +#define __GST_RTP_H265_DEPAY_H__ + +#include +#include +#include +#include + +G_BEGIN_DECLS +#define GST_TYPE_RTP_H265_DEPAY \ + (gst_rtp_h265_depay_get_type()) +#define GST_RTP_H265_DEPAY(obj) \ + (G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_RTP_H265_DEPAY,GstRtpH265Depay)) +#define GST_RTP_H265_DEPAY_CLASS(klass) \ + (G_TYPE_CHECK_CLASS_CAST((klass),GST_TYPE_RTP_H265_DEPAY,GstRtpH265DepayClass)) +#define GST_IS_RTP_H265_DEPAY(obj) \ + (G_TYPE_CHECK_INSTANCE_TYPE((obj),GST_TYPE_RTP_H265_DEPAY)) +#define GST_IS_RTP_H265_DEPAY_CLASS(klass) \ + (G_TYPE_CHECK_CLASS_TYPE((klass),GST_TYPE_RTP_H265_DEPAY)) +typedef struct _GstRtpH265Depay GstRtpH265Depay; +typedef struct _GstRtpH265DepayClass GstRtpH265DepayClass; + +#define GST_H265_VPS_NUT 32 +#define GST_H265_SPS_NUT 33 +#define GST_H265_PPS_NUT 34 + +typedef enum +{ + GST_H265_STREAM_FORMAT_UNKNOWN, + GST_H265_STREAM_FORMAT_BYTESTREAM, + GST_H265_STREAM_FORMAT_HVC1, + GST_H265_STREAM_FORMAT_HEV1 +} GstH265StreamFormat; + +struct _GstRtpH265Depay +{ + GstRTPBaseDepayload depayload; + + gchar *stream_format; + gboolean byte_stream; + + GstBuffer *codec_data; + GstAdapter *adapter; + gboolean wait_start; + + /* nal merging */ + gboolean merge; + GstAdapter *picture_adapter; + gboolean picture_start; + GstClockTime last_ts; + gboolean last_keyframe; + + /* Work around broken payloaders wrt. Fragmentation Units */ + guint8 current_fu_type; + GstClockTime fu_timestamp; + gboolean fu_marker; + + /* misc */ + GPtrArray *vps; + GPtrArray *sps; + GPtrArray *pps; + gboolean new_codec_data; +}; + +struct _GstRtpH265DepayClass +{ + GstRTPBaseDepayloadClass parent_class; +}; + +GType gst_rtp_h265_depay_get_type (void); + +gboolean gst_rtp_h265_depay_plugin_init (GstPlugin * plugin); + +gboolean gst_rtp_h265_add_vps_sps_pps (GstElement * rtph265, GPtrArray * vps, + GPtrArray * sps, GPtrArray * pps, GstBuffer * nal); + +G_END_DECLS +#endif /* __GST_RTP_H265_DEPAY_H__ */ diff --git a/gst/rtp/gstrtph265pay.c b/gst/rtp/gstrtph265pay.c new file mode 100644 index 0000000000..629c6e021f --- /dev/null +++ b/gst/rtp/gstrtph265pay.c @@ -0,0 +1,1497 @@ +/* GStreamer + * Copyright (C) <2006> Wim Taymans + * Copyright (C) <2014> Jurgen Slowack + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifdef HAVE_CONFIG_H +# include "config.h" +#endif + +#include +#include + +#include +#include + +/* Included to not duplicate gst_rtp_h265_add_vps_sps_pps () */ +#include "gstrtph265depay.h" + +#include "gstrtph265pay.h" + + +GST_DEBUG_CATEGORY_STATIC (rtph265pay_debug); +#define GST_CAT_DEFAULT (rtph265pay_debug) + +/* references: + * + * Internet Draft RTP Payload Format for High Efficiency Video Coding + * + * draft-ietf-payload-rtp-h265-03.txt + * + * This draft will be replaced with an RFC, so some details may change. + * + */ + +static GstStaticPadTemplate gst_rtp_h265_pay_sink_template = + GST_STATIC_PAD_TEMPLATE ("sink", + GST_PAD_SINK, + GST_PAD_ALWAYS, + GST_STATIC_CAPS ( + /* Only bytestream format supported for now */ + /* "video/x-h265, " + "stream-format = (string) hvc1, alignment = (string) au; " + "video/x-h265, " + "stream-format = (string) hev1, alignment = (string) au; " */ + "video/x-h265, " + "stream-format = (string) byte-stream, alignment = (string) { nal, au }") + ); + +static GstStaticPadTemplate gst_rtp_h265_pay_src_template = +GST_STATIC_PAD_TEMPLATE ("src", + GST_PAD_SRC, + GST_PAD_ALWAYS, + GST_STATIC_CAPS ("application/x-rtp, " + "media = (string) \"video\", " + "payload = (int) " GST_RTP_PAYLOAD_DYNAMIC_STRING ", " + "clock-rate = (int) 90000, " "encoding-name = (string) \"H265\"") + /** optional parameters **/ + /* "profile-space = (int) [ 0, 3 ], " */ + /* "profile-id = (int) [ 0, 31 ], " */ + /* "tier-flag = (int) [ 0, 1 ], " */ + /* "level-id = (int) [ 0, 255 ], " */ + /* "interop-constraints = (string) ANY, " */ + /* "profile-compatibility-indicator = (string) ANY, " */ + /* "sprop-sub-layer-id = (int) [ 0, 6 ], " */ + /* "recv-sub-layer-id = (int) [ 0, 6 ], " */ + /* "max-recv-level-id = (int) [ 0, 255 ], " */ + /* "tx-mode = (string) {MST , SST}, " */ + /* "sprop-vps = (string) ANY, " */ + /* "sprop-sps = (string) ANY, " */ + /* "sprop-pps = (string) ANY, " */ + /* "sprop-sei = (string) ANY, " */ + /* "max-lsr = (int) ANY, " *//* MUST be in the range of MaxLumaSR to 16 * MaxLumaSR, inclusive */ + /* "max-lps = (int) ANY, " *//* MUST be in the range of MaxLumaPS to 16 * MaxLumaPS, inclusive */ + /* "max-cpb = (int) ANY, " *//* MUST be in the range of MaxCPB to 16 * MaxCPB, inclusive */ + /* "max-dpb = (int) [1, 16], " */ + /* "max-br = (int) ANY, " *//* MUST be in the range of MaxBR to 16 * MaxBR, inclusive, for the highest level */ + /* "max-tr = (int) ANY, " *//* MUST be in the range of MaxTileRows to 16 * MaxTileRows, inclusive, for the highest level */ + /* "max-tc = (int) ANY, " *//* MUST be in the range of MaxTileCols to 16 * MaxTileCols, inclusive, for the highest level */ + /* "max-fps = (int) ANY, " */ + /* "sprop-max-don-diff = (int) [0, 32767], " */ + /* "sprop-depack-buf-nalus = (int) [0, 32767], " */ + /* "sprop-depack-buf-nalus = (int) [0, 4294967295], " */ + /* "depack-buf-cap = (int) [1, 4294967295], " */ + /* "sprop-segmentation-id = (int) [0, 3], " */ + /* "sprop-spatial-segmentation-idc = (string) ANY, " */ + /* "dec-parallel-cap = (string) ANY, " */ + ); + +#define DEFAULT_SPROP_PARAMETER_SETS NULL +#define DEFAULT_CONFIG_INTERVAL 0 + +enum +{ + PROP_0, + PROP_SPROP_PARAMETER_SETS, + PROP_CONFIG_INTERVAL, + PROP_LAST +}; + +#define IS_ACCESS_UNIT(x) (((x) > 0x00) && ((x) < 0x06)) + +static void gst_rtp_h265_pay_finalize (GObject * object); + +static void gst_rtp_h265_pay_set_property (GObject * object, guint prop_id, + const GValue * value, GParamSpec * pspec); +static void gst_rtp_h265_pay_get_property (GObject * object, guint prop_id, + GValue * value, GParamSpec * pspec); + +static GstCaps *gst_rtp_h265_pay_getcaps (GstRTPBasePayload * payload, + GstPad * pad, GstCaps * filter); +static gboolean gst_rtp_h265_pay_setcaps (GstRTPBasePayload * basepayload, + GstCaps * caps); +static GstFlowReturn gst_rtp_h265_pay_handle_buffer (GstRTPBasePayload * pad, + GstBuffer * buffer); +static gboolean gst_rtp_h265_pay_sink_event (GstRTPBasePayload * payload, + GstEvent * event); +static GstStateChangeReturn gst_rtp_h265_pay_change_state (GstElement * + element, GstStateChange transition); + +#define gst_rtp_h265_pay_parent_class parent_class +G_DEFINE_TYPE (GstRtpH265Pay, gst_rtp_h265_pay, GST_TYPE_RTP_BASE_PAYLOAD); + +static void +gst_rtp_h265_pay_class_init (GstRtpH265PayClass * klass) +{ + GObjectClass *gobject_class; + GstElementClass *gstelement_class; + GstRTPBasePayloadClass *gstrtpbasepayload_class; + + gobject_class = (GObjectClass *) klass; + gstelement_class = (GstElementClass *) klass; + gstrtpbasepayload_class = (GstRTPBasePayloadClass *) klass; + + gobject_class->set_property = gst_rtp_h265_pay_set_property; + gobject_class->get_property = gst_rtp_h265_pay_get_property; + + g_object_class_install_property (G_OBJECT_CLASS (klass), + PROP_SPROP_PARAMETER_SETS, g_param_spec_string ("sprop-parameter-sets", + "sprop-parameter-sets", + "The base64 sprop-parameter-sets to set in out caps (set to NULL to " + "extract from stream)", + DEFAULT_SPROP_PARAMETER_SETS, + G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)); + + g_object_class_install_property (G_OBJECT_CLASS (klass), + PROP_CONFIG_INTERVAL, + g_param_spec_uint ("config-interval", + "VPS SPS PPS Send Interval", + "Send VPS, SPS and PPS Insertion Interval in seconds (sprop parameter sets " + "will be multiplexed in the data stream when detected.) (0 = disabled)", + 0, 3600, DEFAULT_CONFIG_INTERVAL, + G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS) + ); + + gobject_class->finalize = gst_rtp_h265_pay_finalize; + + gst_element_class_add_pad_template (gstelement_class, + gst_static_pad_template_get (&gst_rtp_h265_pay_src_template)); + gst_element_class_add_pad_template (gstelement_class, + gst_static_pad_template_get (&gst_rtp_h265_pay_sink_template)); + + gst_element_class_set_static_metadata (gstelement_class, "RTP H265 payloader", + "Codec/Payloader/Network/RTP", + "Payload-encode H265 video into RTP packets (based on draft-ietf-payload-rtp-h265-03.txt)", + "Jurgen Slowack "); + + gstelement_class->change_state = + GST_DEBUG_FUNCPTR (gst_rtp_h265_pay_change_state); + + gstrtpbasepayload_class->get_caps = gst_rtp_h265_pay_getcaps; + gstrtpbasepayload_class->set_caps = gst_rtp_h265_pay_setcaps; + gstrtpbasepayload_class->handle_buffer = gst_rtp_h265_pay_handle_buffer; + gstrtpbasepayload_class->sink_event = gst_rtp_h265_pay_sink_event; + + GST_DEBUG_CATEGORY_INIT (rtph265pay_debug, "rtph265pay", 0, + "H265 RTP Payloader"); +} + +static void +gst_rtp_h265_pay_init (GstRtpH265Pay * rtph265pay) +{ + rtph265pay->queue = g_array_new (FALSE, FALSE, sizeof (guint)); + rtph265pay->profile = 0; + rtph265pay->sps = g_ptr_array_new_with_free_func ( + (GDestroyNotify) gst_buffer_unref); + rtph265pay->pps = g_ptr_array_new_with_free_func ( + (GDestroyNotify) gst_buffer_unref); + rtph265pay->vps = g_ptr_array_new_with_free_func ( + (GDestroyNotify) gst_buffer_unref); + rtph265pay->last_vps_sps_pps = -1; + rtph265pay->vps_sps_pps_interval = DEFAULT_CONFIG_INTERVAL; + + rtph265pay->adapter = gst_adapter_new (); +} + +static void +gst_rtp_h265_pay_clear_vps_sps_pps (GstRtpH265Pay * rtph265pay) +{ + g_ptr_array_set_size (rtph265pay->vps, 0); + g_ptr_array_set_size (rtph265pay->sps, 0); + g_ptr_array_set_size (rtph265pay->pps, 0); +} + +static void +gst_rtp_h265_pay_finalize (GObject * object) +{ + GstRtpH265Pay *rtph265pay; + + rtph265pay = GST_RTP_H265_PAY (object); + + g_array_free (rtph265pay->queue, TRUE); + + g_ptr_array_free (rtph265pay->sps, TRUE); + g_ptr_array_free (rtph265pay->pps, TRUE); + g_ptr_array_free (rtph265pay->vps, TRUE); + + g_free (rtph265pay->sprop_parameter_sets); + + g_object_unref (rtph265pay->adapter); + + G_OBJECT_CLASS (parent_class)->finalize (object); +} + +static const gchar all_levels[][4] = { + "1", + "2", + "2.1", + "3", + "3.1", + "4", + "4.1", + "5", + "5.1", + "5.2", + "6", + "6.1", + "6.2" +}; + +static GstCaps * +gst_rtp_h265_pay_getcaps (GstRTPBasePayload * payload, GstPad * pad, + GstCaps * filter) +{ + GstCaps *template_caps; + GstCaps *allowed_caps; + GstCaps *caps; + GstCaps *icaps; + gboolean append_unrestricted; + guint i; + + allowed_caps = + gst_pad_peer_query_caps (GST_RTP_BASE_PAYLOAD_SRCPAD (payload), NULL); + + if (allowed_caps == NULL) + return NULL; + + template_caps = + gst_static_pad_template_get_caps (&gst_rtp_h265_pay_sink_template); + + if (gst_caps_is_any (allowed_caps)) { + caps = gst_caps_ref (template_caps); + goto done; + } + + if (gst_caps_is_empty (allowed_caps)) { + caps = gst_caps_ref (allowed_caps); + goto done; + } + + caps = gst_caps_new_empty (); + + append_unrestricted = FALSE; + for (i = 0; i < gst_caps_get_size (allowed_caps); i++) { + GstStructure *s = gst_caps_get_structure (allowed_caps, i); + GstStructure *new_s = gst_structure_new_empty ("video/x-h265"); + const gchar *profile_level_id; + + profile_level_id = gst_structure_get_string (s, "profile-level-id"); + + if (profile_level_id && strlen (profile_level_id) == 6) { /* Code taken from gstrtph264pay.c, needs to be revised for H.265 */ + const gchar *profile; + const gchar *level; + long int spsint; + guint8 sps[3]; + + spsint = strtol (profile_level_id, NULL, 16); + sps[0] = spsint >> 16; + sps[1] = spsint >> 8; + sps[2] = spsint; + + profile = gst_codec_utils_h265_get_profile (sps, 3); + level = gst_codec_utils_h265_get_level (sps, 3); + + if (profile && level) { + GST_LOG_OBJECT (payload, "In caps, have profile %s and level %s", + profile, level); + + if (!strcmp (profile, "main")) + gst_structure_set (new_s, "profile", G_TYPE_STRING, profile, NULL); + else { + GValue val = { 0, }; + GValue profiles = { 0, }; + + g_value_init (&profiles, GST_TYPE_LIST); + g_value_init (&val, G_TYPE_STRING); + + g_value_set_static_string (&val, profile); + gst_value_list_append_value (&profiles, &val); + + g_value_set_static_string (&val, "main"); + gst_value_list_append_value (&profiles, &val); + + gst_structure_take_value (new_s, "profile", &profiles); + } + + if (!strcmp (level, "1")) + gst_structure_set (new_s, "level", G_TYPE_STRING, level, NULL); + else { + GValue levels = { 0, }; + GValue val = { 0, }; + int j; + + g_value_init (&levels, GST_TYPE_LIST); + g_value_init (&val, G_TYPE_STRING); + + for (j = 0; j < G_N_ELEMENTS (all_levels); j++) { + g_value_set_static_string (&val, all_levels[j]); + gst_value_list_prepend_value (&levels, &val); + if (!strcmp (level, all_levels[j])) + break; + } + gst_structure_take_value (new_s, "level", &levels); + } + } else { + /* Invalid profile-level-id means main */ + + gst_structure_set (new_s, "profile", G_TYPE_STRING, "main", NULL); + } + } else { + /* No profile-level-id means main or unrestricted */ + + gst_structure_set (new_s, "profile", G_TYPE_STRING, "main", NULL); + append_unrestricted = TRUE; + } + + caps = gst_caps_merge_structure (caps, new_s); + } + + if (append_unrestricted) { + caps = + gst_caps_merge_structure (caps, gst_structure_new ("video/x-h265", NULL, + NULL)); + } + + icaps = gst_caps_intersect (caps, template_caps); + gst_caps_unref (caps); + caps = icaps; + +done: + + gst_caps_unref (template_caps); + gst_caps_unref (allowed_caps); + + GST_LOG_OBJECT (payload, "returning caps %" GST_PTR_FORMAT, caps); + return caps; +} + +/* take the currently configured VPS, SPS and PPS lists and set them on the caps as + * sprop-parameter-sets */ +static gboolean +gst_rtp_h265_pay_set_vps_sps_pps (GstRTPBasePayload * basepayload) +{ + GstRtpH265Pay *payloader = GST_RTP_H265_PAY (basepayload); + gchar *profile; + gchar *set; + GString *sprops; + guint count; + gboolean res; + GstMapInfo map; + guint i; + + sprops = g_string_new (""); + count = 0; + + GST_DEBUG_OBJECT (payloader, + "Entering function gst_rtp_h265_pay_set_vps_sps_pps"); + + /* build the sprop-parameter-sets */ + for (i = 0; i < payloader->vps->len; i++) { + GstBuffer *vps_buf = + GST_BUFFER_CAST (g_ptr_array_index (payloader->vps, i)); + + gst_buffer_map (vps_buf, &map, GST_MAP_READ); + set = g_base64_encode (map.data, map.size); + gst_buffer_unmap (vps_buf, &map); + + g_string_append_printf (sprops, "%s%s", count ? "," : "", set); + g_free (set); + count++; + } + for (i = 0; i < payloader->sps->len; i++) { + GstBuffer *sps_buf = + GST_BUFFER_CAST (g_ptr_array_index (payloader->sps, i)); + + gst_buffer_map (sps_buf, &map, GST_MAP_READ); + set = g_base64_encode (map.data, map.size); + gst_buffer_unmap (sps_buf, &map); + + g_string_append_printf (sprops, "%s%s", count ? "," : "", set); + g_free (set); + count++; + } + for (i = 0; i < payloader->pps->len; i++) { + GstBuffer *pps_buf = + GST_BUFFER_CAST (g_ptr_array_index (payloader->pps, i)); + + gst_buffer_map (pps_buf, &map, GST_MAP_READ); + set = g_base64_encode (map.data, map.size); + gst_buffer_unmap (pps_buf, &map); + + g_string_append_printf (sprops, "%s%s", count ? "," : "", set); + g_free (set); + count++; + } + + if (G_LIKELY (count)) { + /* profile is 24 bit. Force it to respect the limit */ + profile = g_strdup_printf ("%06x", payloader->profile & 0xffffff); + /* combine into output caps */ + res = gst_rtp_base_payload_set_outcaps (basepayload, + "sprop-parameter-sets", G_TYPE_STRING, sprops->str, NULL); + g_free (profile); + } else { + res = gst_rtp_base_payload_set_outcaps (basepayload, NULL); + } + g_string_free (sprops, TRUE); + + return res; +} + + +static gboolean +gst_rtp_h265_pay_setcaps (GstRTPBasePayload * basepayload, GstCaps * caps) +{ + GstRtpH265Pay *rtph265pay; + GstStructure *str; + const GValue *value; + GstMapInfo map; + guint8 *data; + gsize size; + GstBuffer *buffer; + const gchar *alignment, *stream_format; + guint8 num_arrays; + + rtph265pay = GST_RTP_H265_PAY (basepayload); + + str = gst_caps_get_structure (caps, 0); + + /* we can only set the output caps when we found the sprops and profile + * NALs */ + gst_rtp_base_payload_set_options (basepayload, "video", TRUE, "H265", 90000); + + rtph265pay->alignment = GST_H265_ALIGNMENT_UNKNOWN; + alignment = gst_structure_get_string (str, "alignment"); + if (alignment) { + if (g_str_equal (alignment, "au")) + rtph265pay->alignment = GST_H265_ALIGNMENT_AU; + if (g_str_equal (alignment, "nal")) + rtph265pay->alignment = GST_H265_ALIGNMENT_NAL; + } + + rtph265pay->stream_format = GST_H265_STREAM_FORMAT_UNKNOWN; + stream_format = gst_structure_get_string (str, "stream-format"); + if (stream_format) { + if (g_str_equal (stream_format, "hvc1")) + rtph265pay->stream_format = GST_H265_STREAM_FORMAT_HVC1; + if (g_str_equal (stream_format, "hev1")) + rtph265pay->stream_format = GST_H265_STREAM_FORMAT_HEV1; + if (g_str_equal (stream_format, "byte-stream")) + rtph265pay->stream_format = GST_H265_STREAM_FORMAT_BYTESTREAM; + } + + /* packetized HEVC video has a codec_data */ + if ((value = gst_structure_get_value (str, "codec_data"))) { + guint num_vps, num_sps, num_pps; + gint i, j, nal_size; + + GST_DEBUG_OBJECT (rtph265pay, "have packetized h265"); + + buffer = gst_value_get_buffer (value); + + gst_buffer_map (buffer, &map, GST_MAP_READ); + data = map.data; + size = map.size; + + /* parse the hevcC data */ + if (size < 23) + goto hevcc_too_small; + /* HEVCDecoderConfigurationVersion (must be 1) */ + if (data[0] != 1) + goto wrong_version; + + /* profile_space | tier_flag | profile_idc */ + rtph265pay->profile = data[1]; + GST_DEBUG_OBJECT (rtph265pay, "profile %06x", rtph265pay->profile); + + /* profile_compatibility_flags */ + for (i = 2; i < 6; i++) { + for (j = 7; j >= 0; j--) { + GST_DEBUG_OBJECT (rtph265pay, "profile_compatibility_flag %06x", + (data[i] >> j) & 1); + } + } + + GST_DEBUG_OBJECT (rtph265pay, "progressive_source_flag %06x", + (data[6] >> 7) & 1); + GST_DEBUG_OBJECT (rtph265pay, "interlaced_source_flag %06x", + (data[6] >> 6) & 1); + GST_DEBUG_OBJECT (rtph265pay, "non_packed_constraint_flag %06x", + (data[6] >> 5) & 1); + GST_DEBUG_OBJECT (rtph265pay, "frame_only_constraint_flag %06x", + (data[6] >> 4) & 1); + + GST_DEBUG_OBJECT (rtph265pay, "level_idc %06x", data[12]); + + GST_DEBUG_OBJECT (rtph265pay, "min_spatial_segmentation_idc %06x", + ((data[13] ^ 0xf0) << 8) + data[14]); + GST_DEBUG_OBJECT (rtph265pay, "parrallelismType %06x (ignored by paloader)", + data[15]); + + GST_DEBUG_OBJECT (rtph265pay, "sps_chroma_format_idc %06x", + data[16] ^ 0xfc); + GST_DEBUG_OBJECT (rtph265pay, "bit_depth_luma_minus8 %06x", + data[17] ^ 0xf8); + GST_DEBUG_OBJECT (rtph265pay, "bit_depth_chroma_minus8 %06x", + data[18] ^ 0xf8); + GST_DEBUG_OBJECT (rtph265pay, "avgFrameRate %06x", data[19]); + GST_DEBUG_OBJECT (rtph265pay, "avgFrameRate %06x", data[20]); + + /* constFrameRate(2 bits): 0, stream may or may not be of constant framerate + * numTemporalLayers (3 bits): number of temporal layers, value from SPS + * TemporalIdNested (1 bit): sps_temporal_id_nesting_flag from SPS + * lengthSizeMinusOne (2 bits): plus 1 indicates the length of the NALUnitLength */ + GST_DEBUG_OBJECT (rtph265pay, "constFrameRate %06x", + (data[21] >> 6) & 0x03); + GST_DEBUG_OBJECT (rtph265pay, "numTemporalLayers %06x", + (data[21] >> 3) & 0x07); + GST_DEBUG_OBJECT (rtph265pay, "temporal_id_nesting_flag %06x", + (data[21] >> 2) & 0x01); + + rtph265pay->nal_length_size = (data[21] & 0x3) + 1; + GST_DEBUG_OBJECT (rtph265pay, "nal length %u", rtph265pay->nal_length_size); + + num_arrays = GST_READ_UINT8 (data + 22); + + data += 23; + size -= 23; + + if (num_arrays > 0) { + if (data[0] == (0x00 | 0x20)) { /* VPS */ + + data++; + num_vps = data[0] << 8 | data[1]; + data += 2; + size -= 2; + + for (i = 0; i < num_vps; i++) { + + GstBuffer *vps_buf; + + if (size < 2) + goto hevcc_error; + + nal_size = (data[0] << 8) | data[1]; + data += 2; + size -= 2; + + GST_LOG_OBJECT (rtph265pay, "VPS %d size %d", i, nal_size); + + if (size < nal_size) + goto hevcc_error; + + /* make a buffer out of it and add to VPS list */ + vps_buf = gst_buffer_new_and_alloc (nal_size); + gst_buffer_fill (vps_buf, 0, data, nal_size); + gst_rtp_h265_add_vps_sps_pps (GST_ELEMENT (rtph265pay), + rtph265pay->vps, rtph265pay->sps, rtph265pay->pps, vps_buf); + data += nal_size; + size -= nal_size; + } + } + + --num_arrays; + } + + if (num_arrays > 0) { + if (data[0] == (0x00 | 0x21)) { /* SPS */ + + data++; + num_sps = data[0] << 8 | data[1]; + data += 2; + size -= 2; + + for (i = 0; i < num_sps; i++) { + + GstBuffer *sps_buf; + + if (size < 2) + goto hevcc_error; + + nal_size = (data[0] << 8) | data[1]; + data += 2; + size -= 2; + + GST_LOG_OBJECT (rtph265pay, "SPS %d size %d", i, nal_size); + + if (size < nal_size) + goto hevcc_error; + + /* make a buffer out of it and add to SPS list */ + sps_buf = gst_buffer_new_and_alloc (nal_size); + gst_buffer_fill (sps_buf, 0, data, nal_size); + gst_rtp_h265_add_vps_sps_pps (GST_ELEMENT (rtph265pay), + rtph265pay->vps, rtph265pay->sps, rtph265pay->pps, sps_buf); + data += nal_size; + size -= nal_size; + } + } + + --num_arrays; + } + + if (num_arrays > 0) { + if (data[0] == (0x00 | 0x22)) { /* PPS */ + + data++; + num_pps = data[0] << 8 | data[1]; + data += 2; + size -= 2; + + for (i = 0; i < num_pps; i++) { + + GstBuffer *pps_buf; + + if (size < 2) + goto hevcc_error; + + nal_size = (data[0] << 8) | data[1]; + data += 2; + size -= 2; + + GST_LOG_OBJECT (rtph265pay, "PPS %d size %d", i, nal_size); + + if (size < nal_size) + goto hevcc_error; + + /* make a buffer out of it and add to PPS list */ + pps_buf = gst_buffer_new_and_alloc (nal_size); + gst_buffer_fill (pps_buf, 0, data, nal_size); + gst_rtp_h265_add_vps_sps_pps (GST_ELEMENT (rtph265pay), + rtph265pay->vps, rtph265pay->sps, rtph265pay->pps, pps_buf); + data += nal_size; + size -= nal_size; + } + } + + --num_arrays; + } + + /* and update the caps with the collected data */ + if (!gst_rtp_h265_pay_set_vps_sps_pps (basepayload)) + goto set_vps_sps_pps_failed; + + GST_DEBUG_OBJECT (rtph265pay, "Caps have been set"); + + gst_buffer_unmap (buffer, &map); + } else { + GST_DEBUG_OBJECT (rtph265pay, "have bytestream h265"); + } + + return TRUE; + +hevcc_too_small: + { + GST_ERROR_OBJECT (rtph265pay, "hevcC size %" G_GSIZE_FORMAT " < 7", size); + goto error; + } +wrong_version: + { + GST_ERROR_OBJECT (rtph265pay, "wrong hevcC version"); + goto error; + } +hevcc_error: + { + GST_ERROR_OBJECT (rtph265pay, "hevcC too small "); + goto error; + } +set_vps_sps_pps_failed: + { + GST_ERROR_OBJECT (rtph265pay, "failed to set vps/sps/pps"); + goto error; + } +error: + { + gst_buffer_unmap (buffer, &map); + return FALSE; + } +} + +static void +gst_rtp_h265_pay_parse_sprop_parameter_sets (GstRtpH265Pay * rtph265pay) +{ + const gchar *ps; + gchar **params; + guint len; + gint i; + GstBuffer *buf; + + ps = rtph265pay->sprop_parameter_sets; + if (ps == NULL) + return; + + gst_rtp_h265_pay_clear_vps_sps_pps (rtph265pay); + + params = g_strsplit (ps, ",", 0); + len = g_strv_length (params); + + GST_DEBUG_OBJECT (rtph265pay, "we have %d params", len); + + for (i = 0; params[i]; i++) { + gsize nal_len; + GstMapInfo map; + guint8 *nalp; + guint save = 0; + gint state = 0; + + nal_len = strlen (params[i]); + buf = gst_buffer_new_and_alloc (nal_len); + + gst_buffer_map (buf, &map, GST_MAP_WRITE); + nalp = map.data; + nal_len = g_base64_decode_step (params[i], nal_len, nalp, &state, &save); + gst_buffer_unmap (buf, &map); + gst_buffer_resize (buf, 0, nal_len); + + if (!nal_len) { + gst_buffer_unref (buf); + continue; + } + + gst_rtp_h265_add_vps_sps_pps (GST_ELEMENT (rtph265pay), rtph265pay->vps, + rtph265pay->sps, rtph265pay->pps, buf); + } + g_strfreev (params); +} + +static guint +next_start_code (const guint8 * data, guint size) +{ + /* Boyer-Moore string matching algorithm, in a degenerative + * sense because our search 'alphabet' is binary - 0 & 1 only. + * This allow us to simplify the general BM algorithm to a very + * simple form. */ + /* assume 1 is in the 3th byte */ + guint offset = 2; + + while (offset < size) { + if (1 == data[offset]) { + unsigned int shift = offset; + + if (0 == data[--shift]) { + if (0 == data[--shift]) { + return shift; + } + } + /* The jump is always 3 because of the 1 previously matched. + * All the 0's must be after this '1' matched at offset */ + offset += 3; + } else if (0 == data[offset]) { + /* maybe next byte is 1? */ + offset++; + } else { + /* can jump 3 bytes forward */ + offset += 3; + } + /* at each iteration, we rescan in a backward manner until + * we match 0.0.1 in reverse order. Since our search string + * has only 2 'alpabets' (i.e. 0 & 1), we know that any + * mismatch will force us to shift a fixed number of steps */ + } + GST_DEBUG ("Cannot find next NAL start code. returning %u", size); + + return size; +} + +static gboolean +gst_rtp_h265_pay_decode_nal (GstRtpH265Pay * payloader, + const guint8 * data, guint size, GstClockTime dts, GstClockTime pts) +{ + guint8 header, type; + gboolean updated; + + /* default is no update */ + updated = FALSE; + + GST_DEBUG ("NAL payload len=%u", size); + + header = data[0]; + type = header & 0x1f; + + /* We record the timestamp of the last SPS/PPS so + * that we can insert them at regular intervals and when needed. */ + if (GST_H265_NAL_VPS == type || GST_H265_NAL_SPS == type + || GST_H265_NAL_PPS == type) { + GstBuffer *nal; + + /* encode the entire NAL in base64 */ + GST_DEBUG ("Found %s %x %x %x Len=%u", + type == GST_H265_NAL_VPS ? "VPS" : type == + GST_H265_NAL_SPS ? "SPS" : "PPS", (header >> 7), (header >> 5) & 3, + type, size); + + nal = gst_buffer_new_allocate (NULL, size, NULL); + gst_buffer_fill (nal, 0, data, size); + + updated = gst_rtp_h265_add_vps_sps_pps (GST_ELEMENT (payloader), + payloader->vps, payloader->sps, payloader->pps, nal); + + /* remember when we last saw VPS */ + if (updated && pts != -1) + payloader->last_vps_sps_pps = pts; + } else { + GST_DEBUG ("NAL: %x %x %x Len = %u", (header >> 7), + (header >> 5) & 3, type, size); + } + + return updated; +} + +static GstFlowReturn +gst_rtp_h265_pay_payload_nal (GstRTPBasePayload * basepayload, + GstBuffer * paybuf, GstClockTime dts, GstClockTime pts, gboolean end_of_au); + +static GstFlowReturn +gst_rtp_h265_pay_send_vps_sps_pps (GstRTPBasePayload * basepayload, + GstRtpH265Pay * rtph265pay, GstClockTime dts, GstClockTime pts) +{ + GstFlowReturn ret = GST_FLOW_OK; + gboolean sent_all_vps_sps_pps = TRUE; + guint i; + + for (i = 0; i < rtph265pay->vps->len; i++) { + GstBuffer *vps_buf = + GST_BUFFER_CAST (g_ptr_array_index (rtph265pay->vps, i)); + + GST_DEBUG_OBJECT (rtph265pay, "inserting VPS in the stream"); + /* resend VPS */ + ret = gst_rtp_h265_pay_payload_nal (basepayload, gst_buffer_ref (vps_buf), + dts, pts, FALSE); + /* Not critical here; but throw a warning */ + if (ret != GST_FLOW_OK) { + sent_all_vps_sps_pps = FALSE; + GST_WARNING ("Problem pushing VPS"); + } + } + for (i = 0; i < rtph265pay->sps->len; i++) { + GstBuffer *sps_buf = + GST_BUFFER_CAST (g_ptr_array_index (rtph265pay->sps, i)); + + GST_DEBUG_OBJECT (rtph265pay, "inserting SPS in the stream"); + /* resend SPS */ + ret = gst_rtp_h265_pay_payload_nal (basepayload, gst_buffer_ref (sps_buf), + dts, pts, FALSE); + /* Not critical here; but throw a warning */ + if (ret != GST_FLOW_OK) { + sent_all_vps_sps_pps = FALSE; + GST_WARNING ("Problem pushing SPS"); + } + } + for (i = 0; i < rtph265pay->pps->len; i++) { + GstBuffer *pps_buf = + GST_BUFFER_CAST (g_ptr_array_index (rtph265pay->pps, i)); + + GST_DEBUG_OBJECT (rtph265pay, "inserting PPS in the stream"); + /* resend PPS */ + ret = gst_rtp_h265_pay_payload_nal (basepayload, gst_buffer_ref (pps_buf), + dts, pts, FALSE); + /* Not critical here; but throw a warning */ + if (ret != GST_FLOW_OK) { + sent_all_vps_sps_pps = FALSE; + GST_WARNING ("Problem pushing PPS"); + } + } + + if (pts != -1 && sent_all_vps_sps_pps) + rtph265pay->last_vps_sps_pps = pts; + + return ret; +} + +static GstFlowReturn +gst_rtp_h265_pay_payload_nal (GstRTPBasePayload * basepayload, + GstBuffer * paybuf, GstClockTime dts, GstClockTime pts, gboolean end_of_au) +{ + GstRtpH265Pay *rtph265pay; + GstFlowReturn ret; + guint8 nalHeader[2]; + guint8 nalType; + guint packet_len, payload_len, mtu; + GstBuffer *outbuf; + guint8 *payload; + GstBufferList *list = NULL; + gboolean send_vps_sps_pps; + GstRTPBuffer rtp = { NULL }; + guint size = gst_buffer_get_size (paybuf); + + rtph265pay = GST_RTP_H265_PAY (basepayload); + mtu = GST_RTP_BASE_PAYLOAD_MTU (rtph265pay); + + gst_buffer_extract (paybuf, 0, nalHeader, 2); + nalType = (nalHeader[0] >> 1) & 0x3f; + + GST_DEBUG_OBJECT (rtph265pay, "Processing Buffer with NAL TYPE=%d", nalType); + + /* should set src caps before pushing stuff, + * and if we did not see enough VPS/SPS/PPS, that may not be the case */ + if (G_UNLIKELY (!gst_pad_has_current_caps (GST_RTP_BASE_PAYLOAD_SRCPAD + (basepayload)))) + gst_rtp_h265_pay_set_vps_sps_pps (basepayload); + + send_vps_sps_pps = FALSE; + + /* check if we need to emit an VPS/SPS/PPS now */ + if (((nalType == GST_H265_NAL_SLICE_TRAIL_N) + || (nalType == GST_H265_NAL_SLICE_TRAIL_R) + || (nalType == GST_H265_NAL_SLICE_TSA_N) + || (nalType == GST_H265_NAL_SLICE_TSA_R) + || (nalType == GST_H265_NAL_SLICE_STSA_N) + || (nalType == GST_H265_NAL_SLICE_STSA_R) + || (nalType == GST_H265_NAL_SLICE_RASL_N) + || (nalType == GST_H265_NAL_SLICE_RASL_R) + || (nalType == GST_H265_NAL_SLICE_BLA_W_LP) + || (nalType == GST_H265_NAL_SLICE_BLA_W_RADL) + || (nalType == GST_H265_NAL_SLICE_BLA_N_LP) + || (nalType == GST_H265_NAL_SLICE_IDR_W_RADL) + || (nalType == GST_H265_NAL_SLICE_IDR_N_LP) + || (nalType == GST_H265_NAL_SLICE_CRA_NUT)) + && rtph265pay->vps_sps_pps_interval > 0) { + + if (rtph265pay->last_vps_sps_pps != -1) { + guint64 diff; + + GST_LOG_OBJECT (rtph265pay, + "now %" GST_TIME_FORMAT ", last VPS/SPS/PPS %" GST_TIME_FORMAT, + GST_TIME_ARGS (pts), GST_TIME_ARGS (rtph265pay->last_vps_sps_pps)); + + /* calculate diff between last SPS/PPS in milliseconds */ + if (pts > rtph265pay->last_vps_sps_pps) + diff = pts - rtph265pay->last_vps_sps_pps; + else + diff = 0; + + GST_DEBUG_OBJECT (rtph265pay, + "interval since last VPS/SPS/PPS %" GST_TIME_FORMAT, + GST_TIME_ARGS (diff)); + + /* bigger than interval, queue SPS/PPS */ + if (GST_TIME_AS_SECONDS (diff) >= rtph265pay->vps_sps_pps_interval) { + GST_DEBUG_OBJECT (rtph265pay, "time to send VPS/SPS/PPS"); + send_vps_sps_pps = TRUE; + } + } else { + /* no known previous SPS/PPS time, send now */ + GST_DEBUG_OBJECT (rtph265pay, "no previous VPS/SPS/PPS time, send now"); + send_vps_sps_pps = TRUE; + } + } + + if (send_vps_sps_pps || rtph265pay->send_vps_sps_pps) { + /* we need to send SPS/PPS now first. FIXME, don't use the pts for + * checking when we need to send SPS/PPS but convert to running_time first. */ + rtph265pay->send_vps_sps_pps = FALSE; + ret = gst_rtp_h265_pay_send_vps_sps_pps (basepayload, rtph265pay, dts, pts); + if (ret != GST_FLOW_OK) + return ret; + } + + packet_len = gst_rtp_buffer_calc_packet_len (size, 0, 0); + + GST_FIXME_OBJECT (rtph265pay, "Set RTP marker bit appropriately"); + + if (packet_len < mtu) { + GST_DEBUG_OBJECT (rtph265pay, + "NAL Unit fit in one packet datasize=%d mtu=%d", size, mtu); + /* will fit in one packet */ + + /* use buffer lists + * create buffer without payload containing only the RTP header + * (memory block at index 0) */ + outbuf = gst_rtp_buffer_new_allocate (0, 0, 0); + + gst_rtp_buffer_map (outbuf, GST_MAP_WRITE, &rtp); + + /* FIXME : only set the marker bit on packets containing access units */ + /* if (IS_ACCESS_UNIT (nalType) && end_of_au) { + gst_rtp_buffer_set_marker (&rtp, 1); + } */ + + /* timestamp the outbuffer */ + GST_BUFFER_PTS (outbuf) = pts; + GST_BUFFER_DTS (outbuf) = dts; + + /* insert payload memory block */ + outbuf = gst_buffer_append (outbuf, paybuf); + + list = gst_buffer_list_new (); + + /* add the buffer to the buffer list */ + gst_buffer_list_add (list, outbuf); + + gst_rtp_buffer_unmap (&rtp); + + /* push the list to the next element in the pipe */ + ret = gst_rtp_base_payload_push_list (basepayload, list); + } else { + /* fragmentation Units */ + guint limitedSize; + int ii = 0, start = 1, end = 0, pos = 0; + + GST_DEBUG_OBJECT (basepayload, + "NAL Unit DOES NOT fit in one packet datasize=%d mtu=%d", size, mtu); + + pos += 2; + size -= 2; + + ret = GST_FLOW_OK; + + GST_DEBUG_OBJECT (basepayload, "Using FU fragmentation for data size=%d", + size); + + /* We keep 3 bytes for PayloadHdr and FU Header */ + payload_len = gst_rtp_buffer_calc_payload_len (mtu - 3, 0, 0); + + list = gst_buffer_list_new (); + + while (end == 0) { + limitedSize = size < payload_len ? size : payload_len; + GST_DEBUG_OBJECT (basepayload, + "Inside FU fragmentation limitedSize=%d iteration=%d", limitedSize, + ii); + + /* use buffer lists + * create buffer without payload containing only the RTP header + * (memory block at index 0), and with space for PayloadHdr and FU header */ + outbuf = gst_rtp_buffer_new_allocate (3, 0, 0); + + gst_rtp_buffer_map (outbuf, GST_MAP_WRITE, &rtp); + + GST_BUFFER_DTS (outbuf) = dts; + GST_BUFFER_PTS (outbuf) = pts; + payload = gst_rtp_buffer_get_payload (&rtp); + + if (limitedSize == size) { + GST_DEBUG_OBJECT (basepayload, "end size=%d iteration=%d", size, ii); + end = 1; + } + + /* PayloadHdr (type = 49) */ + payload[0] = (nalHeader[0] & 0x81) | (49 << 1); + payload[1] = nalHeader[1]; + + /* FIXME - set RTP marker bit appropriately */ + /* if (IS_ACCESS_UNIT (nalType)) { + gst_rtp_buffer_set_marker (&rtp, end && end_of_au); + } */ + + /* FU Header */ + payload[2] = (start << 7) | (end << 6) | (nalType & 0x3f); + + gst_rtp_buffer_unmap (&rtp); + + /* insert payload memory block */ + gst_buffer_append (outbuf, + gst_buffer_copy_region (paybuf, GST_BUFFER_COPY_MEMORY, pos, + limitedSize)); + + /* add the buffer to the buffer list */ + gst_buffer_list_add (list, outbuf); + + + size -= limitedSize; + pos += limitedSize; + ii++; + start = 0; + } + + ret = gst_rtp_base_payload_push_list (basepayload, list); + gst_buffer_unref (paybuf); + } + return ret; +} + +static GstFlowReturn +gst_rtp_h265_pay_handle_buffer (GstRTPBasePayload * basepayload, + GstBuffer * buffer) +{ + GstRtpH265Pay *rtph265pay; + GstFlowReturn ret; + gsize size; + guint nal_len, i; + GstMapInfo map; + const guint8 *data; + GstClockTime dts, pts; + GArray *nal_queue; + gboolean hevc; + GstBuffer *paybuf = NULL; + gsize skip; + + rtph265pay = GST_RTP_H265_PAY (basepayload); + + /* the input buffer contains one or more NAL units */ + + hevc = (rtph265pay->stream_format == GST_H265_STREAM_FORMAT_HEV1) + || (rtph265pay->stream_format == GST_H265_STREAM_FORMAT_HVC1); + + if (hevc) { + /* In hevc mode, there is no adapter, so nothing to flush */ + if (buffer == NULL) + return GST_FLOW_OK; + gst_buffer_map (buffer, &map, GST_MAP_READ); + data = map.data; + size = map.size; + pts = GST_BUFFER_PTS (buffer); + dts = GST_BUFFER_DTS (buffer); + GST_DEBUG_OBJECT (basepayload, "got %" G_GSIZE_FORMAT " bytes", size); + } else { + dts = gst_adapter_prev_dts (rtph265pay->adapter, NULL); + pts = gst_adapter_prev_pts (rtph265pay->adapter, NULL); + if (buffer) { + if (!GST_CLOCK_TIME_IS_VALID (dts)) + dts = GST_BUFFER_DTS (buffer); + if (!GST_CLOCK_TIME_IS_VALID (pts)) + pts = GST_BUFFER_PTS (buffer); + + gst_adapter_push (rtph265pay->adapter, buffer); + } + size = gst_adapter_available (rtph265pay->adapter); + /* Nothing to do here if the adapter is empty, e.g. on EOS */ + if (size == 0) + return GST_FLOW_OK; + data = gst_adapter_map (rtph265pay->adapter, size); + GST_DEBUG_OBJECT (basepayload, + "got %" G_GSIZE_FORMAT " bytes (%" G_GSIZE_FORMAT ")", size, + buffer ? gst_buffer_get_size (buffer) : 0); + } + + ret = GST_FLOW_OK; + + /* now loop over all NAL units and put them in a packet + * FIXME, we should really try to pack multiple NAL units into one RTP packet + * if we can, especially for the config packets that wont't cause decoder + * latency. */ + if (hevc) { + guint nal_length_size; + gsize offset = 0; + + nal_length_size = rtph265pay->nal_length_size; + + while (size > nal_length_size) { + gint i; + gboolean end_of_au = FALSE; + + nal_len = 0; + for (i = 0; i < nal_length_size; i++) { + nal_len = ((nal_len << 8) + data[i]); + } + + /* skip the length bytes, make sure we don't run past the buffer size */ + data += nal_length_size; + offset += nal_length_size; + size -= nal_length_size; + + if (size >= nal_len) { + GST_DEBUG_OBJECT (basepayload, "got NAL of size %u", nal_len); + } else { + nal_len = size; + GST_DEBUG_OBJECT (basepayload, "got incomplete NAL of size %u", + nal_len); + } + + /* If we're at the end of the buffer, then we're at the end of the + * access unit + */ + if (rtph265pay->alignment == GST_H265_ALIGNMENT_AU + && size - nal_len <= nal_length_size) { + end_of_au = TRUE; + } + + paybuf = gst_buffer_copy_region (buffer, GST_BUFFER_COPY_MEMORY, offset, + nal_len); + + ret = + gst_rtp_h265_pay_payload_nal (basepayload, paybuf, dts, pts, + end_of_au); + if (ret != GST_FLOW_OK) + break; + + data += nal_len; + offset += nal_len; + size -= nal_len; + } + } else { + guint next; + gboolean update = FALSE; + + /* get offset of first start code */ + next = next_start_code (data, size); + + /* skip to start code, if no start code is found, next will be size and we + * will not collect data. */ + data += next; + size -= next; + nal_queue = rtph265pay->queue; + skip = next; + + /* array must be empty when we get here */ + g_assert (nal_queue->len == 0); + + GST_DEBUG_OBJECT (basepayload, + "found first start at %u, bytes left %" G_GSIZE_FORMAT, next, size); + + /* first pass to locate NALs and parse SPS/PPS */ + while (size > 4) { + /* skip start code */ + data += 3; + size -= 3; + + /* use next_start_code() to scan buffer. + * next_start_code() returns the offset in data, + * starting from zero to the first byte of 0.0.0.1 + * If no start code is found, it returns the value of the + * 'size' parameter. + * data is unchanged by the call to next_start_code() + */ + next = next_start_code (data, size); + + if (next == size && buffer != NULL) { + /* Didn't find the start of next NAL and it's not EOS, + * handle it next time */ + break; + } + + /* nal length is distance to next start code */ + nal_len = next; + + GST_DEBUG_OBJECT (basepayload, "found next start at %u of size %u", next, + nal_len); + + if (rtph265pay->sprop_parameter_sets != NULL) { + /* explicitly set profile and sprop, use those */ + if (rtph265pay->update_caps) { + if (!gst_rtp_base_payload_set_outcaps (basepayload, + "sprop-parameter-sets", G_TYPE_STRING, + rtph265pay->sprop_parameter_sets, NULL)) + goto caps_rejected; + + /* parse SPS and PPS from provided parameter set (for insertion) */ + gst_rtp_h265_pay_parse_sprop_parameter_sets (rtph265pay); + + rtph265pay->update_caps = FALSE; + + GST_DEBUG ("outcaps update: sprop-parameter-sets=%s", + rtph265pay->sprop_parameter_sets); + } + } else { + /* We know our stream is a valid H265 NAL packet, + * go parse it for SPS/PPS to enrich the caps */ + /* order: make sure to check nal */ + update = + gst_rtp_h265_pay_decode_nal (rtph265pay, data, nal_len, dts, pts) + || update; + } + /* move to next NAL packet */ + data += nal_len; + size -= nal_len; + + g_array_append_val (nal_queue, nal_len); + } + + /* if has new VPS, SPS & PPS, update the output caps */ + if (G_UNLIKELY (update)) + if (!gst_rtp_h265_pay_set_vps_sps_pps (basepayload)) + goto caps_rejected; + + /* second pass to payload and push */ + + if (nal_queue->len != 0) + gst_adapter_flush (rtph265pay->adapter, skip); + + for (i = 0; i < nal_queue->len; i++) { + guint size; + gboolean end_of_au = FALSE; + + nal_len = g_array_index (nal_queue, guint, i); + /* skip start code */ + gst_adapter_flush (rtph265pay->adapter, 3); + + /* Trim the end unless we're the last NAL in the stream. + * In case we're not at the end of the buffer we know the next block + * starts with 0x000001 so all the 0x00 bytes at the end of this one are + * trailing 0x0 that can be discarded */ + size = nal_len; + data = gst_adapter_map (rtph265pay->adapter, size); + if (i + 1 != nal_queue->len || buffer != NULL) + for (; size > 1 && data[size - 1] == 0x0; size--) + /* skip */ ; + + + /* If it's the last nal unit we have in non-bytestream mode, we can + * assume it's the end of an access-unit + * + * FIXME: We need to wait until the next packet or EOS to + * actually payload the NAL so we can know if the current NAL is + * the last one of an access unit or not if we are in bytestream mode + */ + if ((rtph265pay->alignment == GST_H265_ALIGNMENT_AU || buffer == NULL) && + i == nal_queue->len - 1) + end_of_au = TRUE; + paybuf = gst_adapter_take_buffer (rtph265pay->adapter, size); + g_assert (paybuf); + + /* put the data in one or more RTP packets */ + ret = + gst_rtp_h265_pay_payload_nal (basepayload, paybuf, dts, pts, + end_of_au); + if (ret != GST_FLOW_OK) { + break; + } + + /* move to next NAL packet */ + /* Skips the trailing zeros */ + gst_adapter_flush (rtph265pay->adapter, nal_len - size); + } + g_array_set_size (nal_queue, 0); + } + +done: + if (hevc) { + gst_buffer_unmap (buffer, &map); + gst_buffer_unref (buffer); + } else { + gst_adapter_unmap (rtph265pay->adapter); + } + + return ret; + +caps_rejected: + { + GST_WARNING_OBJECT (basepayload, "Could not set outcaps"); + g_array_set_size (nal_queue, 0); + ret = GST_FLOW_NOT_NEGOTIATED; + goto done; + } +} + +static gboolean +gst_rtp_h265_pay_sink_event (GstRTPBasePayload * payload, GstEvent * event) +{ + gboolean res; + const GstStructure *s; + GstRtpH265Pay *rtph265pay = GST_RTP_H265_PAY (payload); + + switch (GST_EVENT_TYPE (event)) { + case GST_EVENT_FLUSH_STOP: + gst_adapter_clear (rtph265pay->adapter); + break; + case GST_EVENT_CUSTOM_DOWNSTREAM: + s = gst_event_get_structure (event); + if (gst_structure_has_name (s, "GstForceKeyUnit")) { + gboolean resend_codec_data; + + if (gst_structure_get_boolean (s, "all-headers", + &resend_codec_data) && resend_codec_data) + rtph265pay->send_vps_sps_pps = TRUE; + } + break; + case GST_EVENT_EOS: + { + /* call handle_buffer with NULL to flush last NAL from adapter + * in byte-stream mode + */ + gst_rtp_h265_pay_handle_buffer (payload, NULL); + break; + } + case GST_EVENT_STREAM_START: + GST_DEBUG_OBJECT (rtph265pay, + "New stream detected => Clear VPS, SPS and PPS"); + gst_rtp_h265_pay_clear_vps_sps_pps (rtph265pay); + break; + default: + break; + } + + res = GST_RTP_BASE_PAYLOAD_CLASS (parent_class)->sink_event (payload, event); + + return res; +} + +static GstStateChangeReturn +gst_rtp_h265_pay_change_state (GstElement * element, GstStateChange transition) +{ + GstStateChangeReturn ret; + GstRtpH265Pay *rtph265pay = GST_RTP_H265_PAY (element); + + switch (transition) { + case GST_STATE_CHANGE_READY_TO_PAUSED: + rtph265pay->send_vps_sps_pps = FALSE; + gst_adapter_clear (rtph265pay->adapter); + break; + case GST_STATE_CHANGE_PAUSED_TO_READY: + rtph265pay->last_vps_sps_pps = -1; + gst_rtp_h265_pay_clear_vps_sps_pps (rtph265pay); + break; + default: + break; + } + + ret = GST_ELEMENT_CLASS (parent_class)->change_state (element, transition); + + return ret; +} + +static void +gst_rtp_h265_pay_set_property (GObject * object, guint prop_id, + const GValue * value, GParamSpec * pspec) +{ + GstRtpH265Pay *rtph265pay; + + rtph265pay = GST_RTP_H265_PAY (object); + + switch (prop_id) { + case PROP_SPROP_PARAMETER_SETS: + g_free (rtph265pay->sprop_parameter_sets); + rtph265pay->sprop_parameter_sets = g_value_dup_string (value); + rtph265pay->update_caps = TRUE; + break; + case PROP_CONFIG_INTERVAL: + rtph265pay->vps_sps_pps_interval = g_value_get_uint (value); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static void +gst_rtp_h265_pay_get_property (GObject * object, guint prop_id, + GValue * value, GParamSpec * pspec) +{ + GstRtpH265Pay *rtph265pay; + + rtph265pay = GST_RTP_H265_PAY (object); + + switch (prop_id) { + case PROP_SPROP_PARAMETER_SETS: + g_value_set_string (value, rtph265pay->sprop_parameter_sets); + break; + case PROP_CONFIG_INTERVAL: + g_value_set_uint (value, rtph265pay->vps_sps_pps_interval); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +gboolean +gst_rtp_h265_pay_plugin_init (GstPlugin * plugin) +{ + return gst_element_register (plugin, "rtph265pay", + GST_RANK_SECONDARY, GST_TYPE_RTP_H265_PAY); +} diff --git a/gst/rtp/gstrtph265pay.h b/gst/rtp/gstrtph265pay.h new file mode 100644 index 0000000000..8382c5ebef --- /dev/null +++ b/gst/rtp/gstrtph265pay.h @@ -0,0 +1,82 @@ +/* GStreamer + * Copyright (C) <2006> Wim Taymans + * Copyright (C) <2014> Jurgen Slowack + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifndef __GST_RTP_H265_PAY_H__ +#define __GST_RTP_H265_PAY_H__ + +#include +#include +#include +#include + +G_BEGIN_DECLS +#define GST_TYPE_RTP_H265_PAY \ + (gst_rtp_h265_pay_get_type()) +#define GST_RTP_H265_PAY(obj) \ + (G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_RTP_H265_PAY,GstRtpH265Pay)) +#define GST_RTP_H265_PAY_CLASS(klass) \ + (G_TYPE_CHECK_CLASS_CAST((klass),GST_TYPE_RTP_H265_PAY,GstRtpH265PayClass)) +#define GST_IS_RTP_H265_PAY(obj) \ + (G_TYPE_CHECK_INSTANCE_TYPE((obj),GST_TYPE_RTP_H265_PAY)) +#define GST_IS_RTP_H265_PAY_CLASS(klass) \ + (G_TYPE_CHECK_CLASS_TYPE((klass),GST_TYPE_RTP_H265_PAY)) +typedef struct _GstRtpH265Pay GstRtpH265Pay; +typedef struct _GstRtpH265PayClass GstRtpH265PayClass; + +typedef enum +{ + GST_H265_ALIGNMENT_UNKNOWN, + GST_H265_ALIGNMENT_NAL, + GST_H265_ALIGNMENT_AU +} GstH265Alignment; + +struct _GstRtpH265Pay +{ + GstRTPBasePayload payload; + + guint profile; + GPtrArray *sps, *pps, *vps; + + GstH265StreamFormat stream_format; + GstH265Alignment alignment; + guint nal_length_size; + GArray *queue; + + gchar *sprop_parameter_sets; + gboolean update_caps; + + GstAdapter *adapter; + + guint vps_sps_pps_interval; + gboolean send_vps_sps_pps; + GstClockTime last_vps_sps_pps; +}; + +struct _GstRtpH265PayClass +{ + GstRTPBasePayloadClass parent_class; +}; + +GType gst_rtp_h265_pay_get_type (void); + +gboolean gst_rtp_h265_pay_plugin_init (GstPlugin * plugin); + +G_END_DECLS +#endif /* __GST_RTP_H265_PAY_H__ */ From d215b18a209cc742b8ec708c32a1ed3b5a55c433 Mon Sep 17 00:00:00 2001 From: Luis de Bethencourt Date: Thu, 8 Jan 2015 13:47:09 +0000 Subject: [PATCH 03/26] rtp: fix dead code and check for impossible values nal_type is the index for a GstH265NalUnitType enum. There are two types of dead code here: First, after checking if nal_type is >= 39 there are two OR conditionals that check if the value is in ranges higher than that number, so if nal_type >= 39 falls in the True branch those other conditions aren't checked and if it falls in the False branch and they are checked, they will always also be False. They are redundant. Second, the enum has a range of 0 to 40. So the checks for ranges higher than 41 should never be True. Removing this redundant checks. CID 1249684 --- gst/rtp/gstrtph265depay.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/gst/rtp/gstrtph265depay.c b/gst/rtp/gstrtph265depay.c index c25bc8150e..edfdb9bd77 100644 --- a/gst/rtp/gstrtph265depay.c +++ b/gst/rtp/gstrtph265depay.c @@ -914,8 +914,7 @@ gst_rtp_h265_depay_handle_nal (GstRtpH265Depay * rtph265depay, GstBuffer * nal, } complete = TRUE; } else if ((nal_type >= 32 && nal_type <= 35) - || nal_type >= 39 || (nal_type >= 41 && nal_type <= 44) - || (nal_type >= 48 && nal_type <= 55)) { + || nal_type >= 39) { /* VPS, SPS, PPS, SEI, ... terminate an access unit */ complete = TRUE; } From 59fea44503547fa731333235bae115c930375b69 Mon Sep 17 00:00:00 2001 From: Luis de Bethencourt Date: Thu, 8 Jan 2015 15:27:44 +0000 Subject: [PATCH 04/26] rtp: fix nal unit type check After further investigation the previous commit is wrong. The code intended to check if the type is 39 or the ranges 41-44 and 48-55. Just like gsth265parse.c does. Type 40 would not be complete. --- gst/rtp/gstrtph265depay.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/gst/rtp/gstrtph265depay.c b/gst/rtp/gstrtph265depay.c index edfdb9bd77..cb9339daeb 100644 --- a/gst/rtp/gstrtph265depay.c +++ b/gst/rtp/gstrtph265depay.c @@ -914,7 +914,8 @@ gst_rtp_h265_depay_handle_nal (GstRtpH265Depay * rtph265depay, GstBuffer * nal, } complete = TRUE; } else if ((nal_type >= 32 && nal_type <= 35) - || nal_type >= 39) { + || nal_type == 39 || (nal_type >= 41 && nal_type <= 44) + || (nal_type >= 48 && nal_type <= 55)) { /* VPS, SPS, PPS, SEI, ... terminate an access unit */ complete = TRUE; } From e3d8d8cedba098c4c30df82eec67063c9bf2a08f Mon Sep 17 00:00:00 2001 From: Luis de Bethencourt Date: Thu, 8 Jan 2015 15:36:04 +0000 Subject: [PATCH 05/26] rtp: value truncated too short creates dead code type is truncated to 0-31 with "& 0x1f", but right after that it is checks if the value is equivalent to GST_H265_NAL_VPS, GST_H265_NAL_SPS, and GST_H265_NAL_PPS (which are 32, 33, and 34 respectively). Obviously, this will never be True if the value is maximum 31 after the truncation. The intention of the code was to truncate to 0-63. --- gst/rtp/gstrtph265pay.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gst/rtp/gstrtph265pay.c b/gst/rtp/gstrtph265pay.c index 629c6e021f..b2da44d99f 100644 --- a/gst/rtp/gstrtph265pay.c +++ b/gst/rtp/gstrtph265pay.c @@ -821,7 +821,7 @@ gst_rtp_h265_pay_decode_nal (GstRtpH265Pay * payloader, GST_DEBUG ("NAL payload len=%u", size); header = data[0]; - type = header & 0x1f; + type = header & 0x3f; /* We record the timestamp of the last SPS/PPS so * that we can insert them at regular intervals and when needed. */ From 51791d8fe2a4d2fc13575486293b190f07c31a43 Mon Sep 17 00:00:00 2001 From: Luis de Bethencourt Date: Fri, 6 Mar 2015 14:54:41 +0000 Subject: [PATCH 06/26] rtp: donl_present variable unused donl_present is not implemented, yet the value is set and checked a few times. Cleaning this. CID #1249687 --- gst/rtp/gstrtph265depay.c | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/gst/rtp/gstrtph265depay.c b/gst/rtp/gstrtph265depay.c index cb9339daeb..941a5dbf82 100644 --- a/gst/rtp/gstrtph265depay.c +++ b/gst/rtp/gstrtph265depay.c @@ -1040,10 +1040,12 @@ gst_rtp_h265_depay_process (GstRTPBaseDepayload * depayload, GstBuffer * buf) guint outsize, nalu_size; GstClockTime timestamp; gboolean marker; - gboolean donl_present = FALSE; guint8 nuh_layer_id, nuh_temporal_id_plus1; guint8 S, E; guint16 nal_header; +#if 0 + gboolean donl_present = FALSE; +#endif timestamp = GST_BUFFER_TIMESTAMP (buf); @@ -1128,8 +1130,10 @@ gst_rtp_h265_depay_process (GstRTPBaseDepayload * depayload, GstBuffer * buf) rtph265depay->wait_start = FALSE; +#if 0 if (donl_present) goto not_implemented_donl_present; +#endif while (payload_len > 2) { @@ -1207,8 +1211,10 @@ gst_rtp_h265_depay_process (GstRTPBaseDepayload * depayload, GstBuffer * buf) if (rtph265depay->wait_start && !S) goto waiting_start; +#if 0 if (donl_present) goto not_implemented_donl_present; +#endif if (S) { @@ -1288,8 +1294,10 @@ gst_rtp_h265_depay_process (GstRTPBaseDepayload * depayload, GstBuffer * buf) /* All other cases: Single NAL unit packet Section 4.6 */ /* the entire payload is the output buffer */ +#if 0 if (donl_present) goto not_implemented_donl_present; +#endif nalu_size = payload_len; outsize = nalu_size + sizeof (sync_bytes); @@ -1327,6 +1335,7 @@ waiting_start: gst_rtp_buffer_unmap (&rtp); return NULL; } +#if 0 not_implemented_donl_present: { GST_ELEMENT_ERROR (rtph265depay, STREAM, FORMAT, @@ -1334,6 +1343,7 @@ not_implemented_donl_present: gst_rtp_buffer_unmap (&rtp); return NULL; } +#endif not_implemented: { GST_ELEMENT_ERROR (rtph265depay, STREAM, FORMAT, From 693a924461ac535b0fe99b332c9d9d7786b1a975 Mon Sep 17 00:00:00 2001 From: Luis de Bethencourt Date: Fri, 24 Apr 2015 16:48:23 +0100 Subject: [PATCH 07/26] remove unused enum items PROP_LAST This were probably added to the enums due to cargo cult programming and are unused. --- gst/rtp/gstrtph265pay.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/gst/rtp/gstrtph265pay.c b/gst/rtp/gstrtph265pay.c index b2da44d99f..171a242ef5 100644 --- a/gst/rtp/gstrtph265pay.c +++ b/gst/rtp/gstrtph265pay.c @@ -108,8 +108,7 @@ enum { PROP_0, PROP_SPROP_PARAMETER_SETS, - PROP_CONFIG_INTERVAL, - PROP_LAST + PROP_CONFIG_INTERVAL }; #define IS_ACCESS_UNIT(x) (((x) > 0x00) && ((x) < 0x06)) From 7ae49b46ffc7e333993cc229ceb94c5218c0a6f0 Mon Sep 17 00:00:00 2001 From: Luis de Bethencourt Date: Wed, 29 Jul 2015 17:29:28 +0100 Subject: [PATCH 08/26] rtp: remove dead assignment Value set to ret will be overwritten at least once at the end of the while loop, removing assignment. --- gst/rtp/gstrtph265pay.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/gst/rtp/gstrtph265pay.c b/gst/rtp/gstrtph265pay.c index 171a242ef5..0cd8b871ba 100644 --- a/gst/rtp/gstrtph265pay.c +++ b/gst/rtp/gstrtph265pay.c @@ -1047,8 +1047,6 @@ gst_rtp_h265_pay_payload_nal (GstRTPBasePayload * basepayload, pos += 2; size -= 2; - ret = GST_FLOW_OK; - GST_DEBUG_OBJECT (basepayload, "Using FU fragmentation for data size=%d", size); From 470c8b37208ff300f7bddd2868be9ffb04afee7c Mon Sep 17 00:00:00 2001 From: Luis de Bethencourt Date: Wed, 12 Aug 2015 14:59:53 +0100 Subject: [PATCH 09/26] rtph265depay: prevent trying to get 0 bytes from adapter This causes an assertion and would lead to getting a NULL instead of a buffer. Without proper checking this would easily lead to a segfault. Related to rpth264depay: https://bugzilla.gnome.org/show_bug.cgi?id=737199 --- gst/rtp/gstrtph265depay.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/gst/rtp/gstrtph265depay.c b/gst/rtp/gstrtph265depay.c index 941a5dbf82..15599ccd8a 100644 --- a/gst/rtp/gstrtph265depay.c +++ b/gst/rtp/gstrtph265depay.c @@ -1167,10 +1167,12 @@ gst_rtp_h265_depay_process (GstRTPBaseDepayload * depayload, GstBuffer * buf) } outsize = gst_adapter_available (rtph265depay->adapter); - outbuf = gst_adapter_take_buffer (rtph265depay->adapter, outsize); - - outbuf = gst_rtp_h265_depay_handle_nal (rtph265depay, outbuf, timestamp, - marker); + if (outsize > 0) { + outbuf = gst_adapter_take_buffer (rtph265depay->adapter, outsize); + outbuf = + gst_rtp_h265_depay_handle_nal (rtph265depay, outbuf, timestamp, + marker); + } break; } case 49: From a526d014db03a409b6957a8dda596f008fc9bce8 Mon Sep 17 00:00:00 2001 From: Luis de Bethencourt Date: Wed, 12 Aug 2015 15:14:50 +0100 Subject: [PATCH 10/26] rtph265depay: Use GST_BUFFER_PTS() instead of GST_BUFFER_TIMESTAMP() Switching to GST_BUFFER_TIMESTAMP() to be consistent with other rtp code. --- gst/rtp/gstrtph265depay.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gst/rtp/gstrtph265depay.c b/gst/rtp/gstrtph265depay.c index 15599ccd8a..2f0b327b02 100644 --- a/gst/rtp/gstrtph265depay.c +++ b/gst/rtp/gstrtph265depay.c @@ -954,7 +954,7 @@ gst_rtp_h265_depay_handle_nal (GstRtpH265Depay * rtph265depay, GstBuffer * nal, } outbuf = gst_buffer_make_writable (outbuf); - GST_BUFFER_TIMESTAMP (outbuf) = out_timestamp; + GST_BUFFER_PTS (outbuf) = out_timestamp; if (out_keyframe) GST_BUFFER_FLAG_UNSET (outbuf, GST_BUFFER_FLAG_DELTA_UNIT); @@ -1047,7 +1047,7 @@ gst_rtp_h265_depay_process (GstRTPBaseDepayload * depayload, GstBuffer * buf) gboolean donl_present = FALSE; #endif - timestamp = GST_BUFFER_TIMESTAMP (buf); + timestamp = GST_BUFFER_PTS (buf); gst_rtp_buffer_map (buf, GST_MAP_READ, &rtp); From 0bfa97b047c788875c9cb1f4a570c5db8c2242ce Mon Sep 17 00:00:00 2001 From: Luis de Bethencourt Date: Wed, 12 Aug 2015 15:49:50 +0100 Subject: [PATCH 11/26] rtph265depay: implement process_rtp_packet() vfunc For more optimised RTP packet handling: means we don't need to map the input buffer again but can just re-use the mapping the base class has already done. Based on: https://bugzilla.gnome.org/show_bug.cgi?id=750235 https://bugzilla.gnome.org/show_bug.cgi?id=753228 --- gst/rtp/gstrtph265depay.c | 24 ++++++++---------------- 1 file changed, 8 insertions(+), 16 deletions(-) diff --git a/gst/rtp/gstrtph265depay.c b/gst/rtp/gstrtph265depay.c index 2f0b327b02..39e6c6886f 100644 --- a/gst/rtp/gstrtph265depay.c +++ b/gst/rtp/gstrtph265depay.c @@ -103,7 +103,7 @@ static GstStateChangeReturn gst_rtp_h265_depay_change_state (GstElement * element, GstStateChange transition); static GstBuffer *gst_rtp_h265_depay_process (GstRTPBaseDepayload * depayload, - GstBuffer * buf); + GstRTPBuffer * rtp); static gboolean gst_rtp_h265_depay_setcaps (GstRTPBaseDepayload * filter, GstCaps * caps); static gboolean gst_rtp_h265_depay_handle_event (GstRTPBaseDepayload * depay, @@ -133,7 +133,7 @@ gst_rtp_h265_depay_class_init (GstRtpH265DepayClass * klass) "Jurgen Slowack "); gstelement_class->change_state = gst_rtp_h265_depay_change_state; - gstrtpbasedepayload_class->process = gst_rtp_h265_depay_process; + gstrtpbasedepayload_class->process_rtp_packet = gst_rtp_h265_depay_process; gstrtpbasedepayload_class->set_caps = gst_rtp_h265_depay_setcaps; gstrtpbasedepayload_class->handle_event = gst_rtp_h265_depay_handle_event; } @@ -1016,17 +1016,16 @@ not_implemented: } static GstBuffer * -gst_rtp_h265_depay_process (GstRTPBaseDepayload * depayload, GstBuffer * buf) +gst_rtp_h265_depay_process (GstRTPBaseDepayload * depayload, GstRTPBuffer * rtp) { GstRtpH265Depay *rtph265depay; GstBuffer *outbuf = NULL; guint8 nal_unit_type; - GstRTPBuffer rtp = { NULL }; rtph265depay = GST_RTP_H265_DEPAY (depayload); /* flush remaining data on discont */ - if (GST_BUFFER_IS_DISCONT (buf)) { + if (GST_BUFFER_IS_DISCONT (rtp->buffer)) { gst_adapter_clear (rtph265depay->adapter); rtph265depay->wait_start = TRUE; rtph265depay->current_fu_type = 0; @@ -1047,13 +1046,11 @@ gst_rtp_h265_depay_process (GstRTPBaseDepayload * depayload, GstBuffer * buf) gboolean donl_present = FALSE; #endif - timestamp = GST_BUFFER_PTS (buf); + timestamp = GST_BUFFER_PTS (rtp->buffer); - gst_rtp_buffer_map (buf, GST_MAP_READ, &rtp); - - payload_len = gst_rtp_buffer_get_payload_len (&rtp); - payload = gst_rtp_buffer_get_payload (&rtp); - marker = gst_rtp_buffer_get_marker (&rtp); + payload_len = gst_rtp_buffer_get_payload_len (rtp); + payload = gst_rtp_buffer_get_payload (rtp); + marker = gst_rtp_buffer_get_marker (rtp); GST_DEBUG_OBJECT (rtph265depay, "receiving %d bytes", payload_len); @@ -1319,7 +1316,6 @@ gst_rtp_h265_depay_process (GstRTPBaseDepayload * depayload, GstBuffer * buf) break; } } - gst_rtp_buffer_unmap (&rtp); } return outbuf; @@ -1328,13 +1324,11 @@ gst_rtp_h265_depay_process (GstRTPBaseDepayload * depayload, GstBuffer * buf) empty_packet: { GST_DEBUG_OBJECT (rtph265depay, "empty packet"); - gst_rtp_buffer_unmap (&rtp); return NULL; } waiting_start: { GST_DEBUG_OBJECT (rtph265depay, "waiting for start"); - gst_rtp_buffer_unmap (&rtp); return NULL; } #if 0 @@ -1342,7 +1336,6 @@ not_implemented_donl_present: { GST_ELEMENT_ERROR (rtph265depay, STREAM, FORMAT, (NULL), ("DONL field present not supported yet")); - gst_rtp_buffer_unmap (&rtp); return NULL; } #endif @@ -1350,7 +1343,6 @@ not_implemented: { GST_ELEMENT_ERROR (rtph265depay, STREAM, FORMAT, (NULL), ("NAL unit type %d not supported yet", nal_unit_type)); - gst_rtp_buffer_unmap (&rtp); return NULL; } } From d10b6f1e3ac35c48ca9239d8d526e244d6b982f3 Mon Sep 17 00:00:00 2001 From: Luis de Bethencourt Date: Wed, 12 Aug 2015 16:11:00 +0100 Subject: [PATCH 12/26] rtph265depay: Insert SPS/PPS NALs into the stream rtph264depay does the same and this fixes decoding of some streams with 32 SPS (or 256 PPS). It is allowed to have SPS ID 0 to 31 (or PPS ID 0 to 255), but the field in the codec_data for the number of SPS or PPS is only 5 (or 8) bit. As such, 32 SPS (or 256 PPS) are interpreted as 0 everywhere. This looks like a mistake in the part of the spect about the codec_data. --- gst/rtp/gstrtph265depay.c | 50 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/gst/rtp/gstrtph265depay.c b/gst/rtp/gstrtph265depay.c index 39e6c6886f..de4e20292e 100644 --- a/gst/rtp/gstrtph265depay.c +++ b/gst/rtp/gstrtph265depay.c @@ -534,6 +534,56 @@ gst_rtp_h265_set_src_caps (GstRtpH265Depay * rtph265depay) srccaps); gst_caps_unref (srccaps); + /* Insert SPS and PPS into the stream on next opportunity */ + { + gint i; + GstBuffer *codec_data; + GstMapInfo map; + guint8 *data; + guint len = 0; + + for (i = 0; i < rtph265depay->sps->len; i++) { + len += 4 + gst_buffer_get_size (g_ptr_array_index (rtph265depay->sps, i)); + } + + for (i = 0; i < rtph265depay->pps->len; i++) { + len += 4 + gst_buffer_get_size (g_ptr_array_index (rtph265depay->pps, i)); + } + + codec_data = gst_buffer_new_and_alloc (len); + gst_buffer_map (codec_data, &map, GST_MAP_WRITE); + data = map.data; + + for (i = 0; i < rtph265depay->sps->len; i++) { + GstBuffer *sps_buf = g_ptr_array_index (rtph265depay->sps, i); + guint sps_size = gst_buffer_get_size (sps_buf); + + if (rtph265depay->byte_stream) + memcpy (data, sync_bytes, sizeof (sync_bytes)); + else + GST_WRITE_UINT32_BE (data, sps_size); + gst_buffer_extract (sps_buf, 0, data + 4, -1); + data += 4 + sps_size; + } + + for (i = 0; i < rtph265depay->pps->len; i++) { + GstBuffer *pps_buf = g_ptr_array_index (rtph265depay->pps, i); + guint pps_size = gst_buffer_get_size (pps_buf); + + if (rtph265depay->byte_stream) + memcpy (data, sync_bytes, sizeof (sync_bytes)); + else + GST_WRITE_UINT32_BE (data, pps_size); + gst_buffer_extract (pps_buf, 0, data + 4, -1); + data += 4 + pps_size; + } + + gst_buffer_unmap (codec_data, &map); + if (rtph265depay->codec_data) + gst_buffer_unref (rtph265depay->codec_data); + rtph265depay->codec_data = codec_data; + } + if (res) rtph265depay->new_codec_data = FALSE; From 3979ffa6a3ad3c047d4d1d4fe36a36acd500ee42 Mon Sep 17 00:00:00 2001 From: Luis de Bethencourt Date: Wed, 12 Aug 2015 16:43:48 +0100 Subject: [PATCH 13/26] rtph265depay: PPS replaces old PPS if it has the same id https://bugzilla.gnome.org/show_bug.cgi?id=753228 --- gst/rtp/gstrtph265depay.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gst/rtp/gstrtph265depay.c b/gst/rtp/gstrtph265depay.c index de4e20292e..5ee4a74e79 100644 --- a/gst/rtp/gstrtph265depay.c +++ b/gst/rtp/gstrtph265depay.c @@ -686,7 +686,7 @@ gst_rtp_h265_add_vps_sps_pps (GstElement * rtph265, GPtrArray * vps_array, gst_buffer_map (pps, &ppsmap, GST_MAP_READ); parse_pps (&ppsmap, &tmp_sps_id, &tmp_pps_id); - if (sps_id == tmp_sps_id && pps_id == tmp_pps_id) { + if (pps_id == tmp_pps_id) { if (map.size == ppsmap.size && memcmp (map.data, ppsmap.data, ppsmap.size) == 0) { GST_LOG_OBJECT (rtph265, "Unchanged PPS %u:%u, not updating", sps_id, From 18b628824b7b3aaaf847efba74e049027df05ac3 Mon Sep 17 00:00:00 2001 From: Luis de Bethencourt Date: Wed, 12 Aug 2015 17:22:42 +0100 Subject: [PATCH 14/26] rtph265depay: only update the srcpad caps if something else than the codec_data changed h264parse and gstrtph264depay do the same, let's keep the behaviour consistent. As we now include the codec_data inside the stream, this causes less caps renegotiation. https://bugzilla.gnome.org/show_bug.cgi?id=753228 --- gst/rtp/gstrtph265depay.c | 43 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 41 insertions(+), 2 deletions(-) diff --git a/gst/rtp/gstrtph265depay.c b/gst/rtp/gstrtph265depay.c index 5ee4a74e79..99755d7d2c 100644 --- a/gst/rtp/gstrtph265depay.c +++ b/gst/rtp/gstrtph265depay.c @@ -530,8 +530,47 @@ gst_rtp_h265_set_src_caps (GstRtpH265Depay * rtph265depay) gst_buffer_unref (codec_data); } - res = gst_pad_set_caps (GST_RTP_BASE_DEPAYLOAD_SRCPAD (rtph265depay), - srccaps); + if (gst_pad_has_current_caps (GST_RTP_BASE_DEPAYLOAD_SRCPAD (rtph265depay))) { + GstCaps *old_caps = + gst_pad_get_current_caps (GST_RTP_BASE_DEPAYLOAD_SRCPAD (rtph265depay)); + + /* Only update the caps if they are not equal. For + * AVC we don't update caps if only the codec_data + * changes. This is the same behaviour as in h264parse + * and gstrtph264depay + */ + if (rtph265depay->byte_stream) { + if (!gst_caps_is_equal (srccaps, old_caps)) + res = + gst_pad_set_caps (GST_RTP_BASE_DEPAYLOAD_SRCPAD (rtph265depay), + srccaps); + else + res = TRUE; + } else { + GstCaps *tmp_caps = gst_caps_copy (srccaps); + GstStructure *old_s, *tmp_s; + + old_s = gst_caps_get_structure (old_caps, 0); + tmp_s = gst_caps_get_structure (tmp_caps, 0); + if (gst_structure_has_field (old_s, "codec_data")) + gst_structure_set_value (tmp_s, "codec_data", + gst_structure_get_value (old_s, "codec_data")); + + if (!gst_caps_is_equal (old_caps, tmp_caps)) + res = + gst_pad_set_caps (GST_RTP_BASE_DEPAYLOAD_SRCPAD (rtph265depay), + srccaps); + else + res = TRUE; + + gst_caps_unref (tmp_caps); + } + } else { + res = + gst_pad_set_caps (GST_RTP_BASE_DEPAYLOAD_SRCPAD (rtph265depay), + srccaps); + } + gst_caps_unref (srccaps); /* Insert SPS and PPS into the stream on next opportunity */ From 3bede1c95beba530694afc76e017e574376bbe92 Mon Sep 17 00:00:00 2001 From: Luis de Bethencourt Date: Wed, 12 Aug 2015 17:54:52 +0100 Subject: [PATCH 15/26] rtph265depay: checking if depay has sps/pps nals before insertion Related to: https://bugzilla.gnome.org/show_bug.cgi?id=753430 https://bugzilla.gnome.org/show_bug.cgi?id=753228 --- gst/rtp/gstrtph265depay.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gst/rtp/gstrtph265depay.c b/gst/rtp/gstrtph265depay.c index 99755d7d2c..7d573fe9ef 100644 --- a/gst/rtp/gstrtph265depay.c +++ b/gst/rtp/gstrtph265depay.c @@ -574,7 +574,7 @@ gst_rtp_h265_set_src_caps (GstRtpH265Depay * rtph265depay) gst_caps_unref (srccaps); /* Insert SPS and PPS into the stream on next opportunity */ - { + if (rtph265depay->sps->len > 0 || rtph265depay->pps->len > 0) { gint i; GstBuffer *codec_data; GstMapInfo map; From f1e284943886fd282aae3fe83c267e2b9623b04e Mon Sep 17 00:00:00 2001 From: Luis de Bethencourt Date: Fri, 14 Aug 2015 11:49:51 +0100 Subject: [PATCH 16/26] rtph265depay: copy metadata in the depayloader, but only the relevant ones The payloader didn't copy anything so far, the depayloader copied every possible meta. Let's make it consistent and just copy all metas without tags or with only the video tag. https://bugzilla.gnome.org/show_bug.cgi?id=751774 --- gst/rtp/gstrtph265depay.c | 80 +++++++++++++++++++++++++++++++++++++++ gst/rtp/gstrtph265depay.h | 9 +++++ 2 files changed, 89 insertions(+) diff --git a/gst/rtp/gstrtph265depay.c b/gst/rtp/gstrtph265depay.c index 7d573fe9ef..9718d5831a 100644 --- a/gst/rtp/gstrtph265depay.c +++ b/gst/rtp/gstrtph265depay.c @@ -27,6 +27,7 @@ #include #include +#include #include "gstrtph265depay.h" GST_DEBUG_CATEGORY_STATIC (rtph265depay_debug); @@ -938,6 +939,60 @@ gst_rtp_h265_complete_au (GstRtpH265Depay * rtph265depay, #define NAL_TYPE_IS_KEY(nt) (NAL_TYPE_IS_PARAMETER_SET(nt) || NAL_TYPE_IS_CODED_SLICE_SEGMENT(nt)) +static gboolean +foreach_metadata_copy (GstBuffer * inbuf, GstMeta ** meta, gpointer user_data) +{ + CopyMetaData *data = user_data; + GstElement *element = data->element; + GstBuffer *outbuf = data->outbuf; + const GstMetaInfo *info = (*meta)->info; + const gchar *const *tags = gst_meta_api_type_get_tags (info->api); + + if (!tags || (g_strv_length ((gchar **) tags) == 1 + && gst_meta_api_type_has_tag (info->api, + g_quark_from_string (GST_META_TAG_VIDEO_STR)))) { + GstMetaTransformCopy copy_data = { FALSE, 0, -1 }; + GST_DEBUG_OBJECT (element, "copy metadata %s", g_type_name (info->api)); + /* simply copy then */ + info->transform_func (outbuf, *meta, inbuf, + _gst_meta_transform_copy, ©_data); + } else { + GST_DEBUG_OBJECT (element, "not copying metadata %s", + g_type_name (info->api)); + } + + return TRUE; +} + +/* TODO: Should probably make copy_tag an array at some point */ +void +gst_rtp_copy_meta (GstElement * element, GstBuffer * outbuf, GstBuffer * inbuf, + GQuark copy_tag) +{ + CopyMetaData data = { element, outbuf, copy_tag }; + + gst_buffer_foreach_meta (inbuf, foreach_metadata_copy, &data); +} + +static gboolean +foreach_metadata_drop (GstBuffer * inbuf, GstMeta ** meta, gpointer user_data) +{ + GstRtpH265Depay *depay = user_data; + const GstMetaInfo *info = (*meta)->info; + const gchar *const *tags = gst_meta_api_type_get_tags (info->api); + + if (!tags || (g_strv_length ((gchar **) tags) == 1 + && gst_meta_api_type_has_tag (info->api, + g_quark_from_string (GST_META_TAG_VIDEO_STR)))) { + GST_DEBUG_OBJECT (depay, "keeping metadata %s", g_type_name (info->api)); + } else { + GST_DEBUG_OBJECT (depay, "dropping metadata %s", g_type_name (info->api)); + *meta = NULL; + } + + return TRUE; +} + static GstBuffer * gst_rtp_h265_depay_handle_nal (GstRtpH265Depay * rtph265depay, GstBuffer * nal, GstClockTime in_timestamp, gboolean marker) @@ -1037,12 +1092,17 @@ gst_rtp_h265_depay_handle_nal (GstRtpH265Depay * rtph265depay, GstBuffer * nal, /* prepend codec_data */ if (rtph265depay->codec_data) { GST_DEBUG_OBJECT (depayload, "prepending codec_data"); + gst_rtp_copy_meta (GST_ELEMENT_CAST (rtph265depay), + rtph265depay->codec_data, outbuf, + g_quark_from_static_string (GST_META_TAG_VIDEO_STR)); outbuf = gst_buffer_append (rtph265depay->codec_data, outbuf); rtph265depay->codec_data = NULL; out_keyframe = TRUE; } outbuf = gst_buffer_make_writable (outbuf); + gst_buffer_foreach_meta (outbuf, foreach_metadata_drop, depayload); + GST_BUFFER_PTS (outbuf) = out_timestamp; if (out_keyframe) @@ -1108,6 +1168,7 @@ static GstBuffer * gst_rtp_h265_depay_process (GstRTPBaseDepayload * depayload, GstRTPBuffer * rtp) { GstRtpH265Depay *rtph265depay; + GstBuffer *buf; GstBuffer *outbuf = NULL; guint8 nal_unit_type; @@ -1139,6 +1200,7 @@ gst_rtp_h265_depay_process (GstRTPBaseDepayload * depayload, GstRTPBuffer * rtp) payload_len = gst_rtp_buffer_get_payload_len (rtp); payload = gst_rtp_buffer_get_payload (rtp); + buf = gst_rtp_buffer_get_payload_buffer (rtp); marker = gst_rtp_buffer_get_marker (rtp); GST_DEBUG_OBJECT (rtph265depay, "receiving %d bytes", payload_len); @@ -1246,6 +1308,9 @@ gst_rtp_h265_depay_process (GstRTPBaseDepayload * depayload, GstRTPBuffer * rtp) memcpy (map.data + sizeof (sync_bytes), payload, nalu_size); gst_buffer_unmap (outbuf, &map); + gst_rtp_copy_meta (GST_ELEMENT_CAST (rtph265depay), outbuf, buf, + g_quark_from_static_string (GST_META_TAG_VIDEO_STR)); + gst_adapter_push (rtph265depay->adapter, outbuf); payload += nalu_size; @@ -1340,6 +1405,9 @@ gst_rtp_h265_depay_process (GstRTPBaseDepayload * depayload, GstRTPBuffer * rtp) map.data[sizeof (sync_bytes) + 1] = nal_header & 0xff; gst_buffer_unmap (outbuf, &map); + gst_rtp_copy_meta (GST_ELEMENT_CAST (rtph265depay), outbuf, buf, + g_quark_from_static_string (GST_META_TAG_VIDEO_STR)); + GST_DEBUG_OBJECT (rtph265depay, "queueing %d bytes", outsize); /* and assemble in the adapter */ @@ -1357,6 +1425,9 @@ gst_rtp_h265_depay_process (GstRTPBaseDepayload * depayload, GstRTPBuffer * rtp) outbuf = gst_buffer_new_and_alloc (outsize); gst_buffer_fill (outbuf, 0, payload, outsize); + gst_rtp_copy_meta (GST_ELEMENT_CAST (rtph265depay), outbuf, buf, + g_quark_from_static_string (GST_META_TAG_VIDEO_STR)); + GST_DEBUG_OBJECT (rtph265depay, "queueing %d bytes", outsize); /* and assemble in the adapter */ @@ -1400,6 +1471,9 @@ gst_rtp_h265_depay_process (GstRTPBaseDepayload * depayload, GstRTPBuffer * rtp) memcpy (map.data + sizeof (sync_bytes), payload, nalu_size); gst_buffer_unmap (outbuf, &map); + gst_rtp_copy_meta (GST_ELEMENT_CAST (rtph265depay), outbuf, buf, + g_quark_from_static_string (GST_META_TAG_VIDEO_STR)); + outbuf = gst_rtp_h265_depay_handle_nal (rtph265depay, outbuf, timestamp, marker); break; @@ -1407,17 +1481,21 @@ gst_rtp_h265_depay_process (GstRTPBaseDepayload * depayload, GstRTPBuffer * rtp) } } + gst_buffer_unref (buf); + return outbuf; /* ERRORS */ empty_packet: { GST_DEBUG_OBJECT (rtph265depay, "empty packet"); + gst_buffer_unref (buf); return NULL; } waiting_start: { GST_DEBUG_OBJECT (rtph265depay, "waiting for start"); + gst_buffer_unref (buf); return NULL; } #if 0 @@ -1425,6 +1503,7 @@ not_implemented_donl_present: { GST_ELEMENT_ERROR (rtph265depay, STREAM, FORMAT, (NULL), ("DONL field present not supported yet")); + gst_buffer_unref (buf); return NULL; } #endif @@ -1432,6 +1511,7 @@ not_implemented: { GST_ELEMENT_ERROR (rtph265depay, STREAM, FORMAT, (NULL), ("NAL unit type %d not supported yet", nal_unit_type)); + gst_buffer_unref (buf); return NULL; } } diff --git a/gst/rtp/gstrtph265depay.h b/gst/rtp/gstrtph265depay.h index ed49329a3d..9f50ac60e5 100644 --- a/gst/rtp/gstrtph265depay.h +++ b/gst/rtp/gstrtph265depay.h @@ -87,6 +87,15 @@ struct _GstRtpH265DepayClass GstRTPBaseDepayloadClass parent_class; }; +typedef struct +{ + GstElement *element; + GstBuffer *outbuf; + GQuark copy_tag; +} CopyMetaData; + +void gst_rtp_copy_meta (GstElement * element, GstBuffer *outbuf, GstBuffer *inbuf, GQuark copy_tag); + GType gst_rtp_h265_depay_get_type (void); gboolean gst_rtp_h265_depay_plugin_init (GstPlugin * plugin); From f2bae3ab5936271b12f92aefdd0cba559c79ecab Mon Sep 17 00:00:00 2001 From: Luis de Bethencourt Date: Fri, 14 Aug 2015 15:08:08 +0100 Subject: [PATCH 17/26] rtph265pay: fix buffer leak when using SPS/PPS Fixes a buffer leak that would occur if the pipeline was shutdown while a SPS/PPS header was being created. https://bugzilla.gnome.org/show_bug.cgi?id=741271 --- gst/rtp/gstrtph265pay.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/gst/rtp/gstrtph265pay.c b/gst/rtp/gstrtph265pay.c index 0cd8b871ba..0a5bac611f 100644 --- a/gst/rtp/gstrtph265pay.c +++ b/gst/rtp/gstrtph265pay.c @@ -995,8 +995,10 @@ gst_rtp_h265_pay_payload_nal (GstRTPBasePayload * basepayload, * checking when we need to send SPS/PPS but convert to running_time first. */ rtph265pay->send_vps_sps_pps = FALSE; ret = gst_rtp_h265_pay_send_vps_sps_pps (basepayload, rtph265pay, dts, pts); - if (ret != GST_FLOW_OK) + if (ret != GST_FLOW_OK) { + gst_buffer_unref (paybuf); return ret; + } } packet_len = gst_rtp_buffer_calc_packet_len (size, 0, 0); From df724c410b02b82bd7db893d24e8572a06c2fcb1 Mon Sep 17 00:00:00 2001 From: Luis de Bethencourt Date: Sat, 15 Aug 2015 11:30:36 +0100 Subject: [PATCH 18/26] rtph265pay: fix potential crash when shutting down A race condition in the state change function may cause buffers to be unreffed while they are still used by the streaming thread in gst_rtp_h265_pay_send_vps_sps_pps() resulting in a crash. Chain up to the parent class first in the state change function to make sure streaming has stopped and only then free those buffers. https://bugzilla.gnome.org/show_bug.cgi?id=741381 --- gst/rtp/gstrtph265pay.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/gst/rtp/gstrtph265pay.c b/gst/rtp/gstrtph265pay.c index 0a5bac611f..1ed17f88f2 100644 --- a/gst/rtp/gstrtph265pay.c +++ b/gst/rtp/gstrtph265pay.c @@ -1431,6 +1431,13 @@ gst_rtp_h265_pay_change_state (GstElement * element, GstStateChange transition) rtph265pay->send_vps_sps_pps = FALSE; gst_adapter_clear (rtph265pay->adapter); break; + default: + break; + } + + ret = GST_ELEMENT_CLASS (parent_class)->change_state (element, transition); + + switch (transition) { case GST_STATE_CHANGE_PAUSED_TO_READY: rtph265pay->last_vps_sps_pps = -1; gst_rtp_h265_pay_clear_vps_sps_pps (rtph265pay); @@ -1439,8 +1446,6 @@ gst_rtp_h265_pay_change_state (GstElement * element, GstStateChange transition) break; } - ret = GST_ELEMENT_CLASS (parent_class)->change_state (element, transition); - return ret; } From 8611645af66a66326fd081adf209d025e1050a6f Mon Sep 17 00:00:00 2001 From: Luis de Bethencourt Date: Sat, 15 Aug 2015 11:41:40 +0100 Subject: [PATCH 19/26] rtph265pay: Use GST_WARNING_OBJECT() instead of GST_WARNING() https://bugzilla.gnome.org/show_bug.cgi?id=753228 --- gst/rtp/gstrtph265pay.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gst/rtp/gstrtph265pay.c b/gst/rtp/gstrtph265pay.c index 1ed17f88f2..549deb8d97 100644 --- a/gst/rtp/gstrtph265pay.c +++ b/gst/rtp/gstrtph265pay.c @@ -874,7 +874,7 @@ gst_rtp_h265_pay_send_vps_sps_pps (GstRTPBasePayload * basepayload, /* Not critical here; but throw a warning */ if (ret != GST_FLOW_OK) { sent_all_vps_sps_pps = FALSE; - GST_WARNING ("Problem pushing VPS"); + GST_WARNING_OBJECT (basepayload, "Problem pushing VPS"); } } for (i = 0; i < rtph265pay->sps->len; i++) { @@ -888,7 +888,7 @@ gst_rtp_h265_pay_send_vps_sps_pps (GstRTPBasePayload * basepayload, /* Not critical here; but throw a warning */ if (ret != GST_FLOW_OK) { sent_all_vps_sps_pps = FALSE; - GST_WARNING ("Problem pushing SPS"); + GST_WARNING_OBJECT (basepayload, "Problem pushing SPS"); } } for (i = 0; i < rtph265pay->pps->len; i++) { From 1e55d0d725c4f86fddcbbf3663f1edc65b6aae54 Mon Sep 17 00:00:00 2001 From: Luis de Bethencourt Date: Sat, 15 Aug 2015 14:45:34 +0100 Subject: [PATCH 20/26] rtph265pay: Copy metadata in the payloader, but only the relevant ones The payloader didn't copy anything so far, the depayloader copied every possible meta. Let's make it consistent and just copy all metas without tags or with only the video tag. https://bugzilla.gnome.org/show_bug.cgi?id=751774 --- gst/rtp/gstrtph265pay.c | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/gst/rtp/gstrtph265pay.c b/gst/rtp/gstrtph265pay.c index 549deb8d97..b17b091618 100644 --- a/gst/rtp/gstrtph265pay.c +++ b/gst/rtp/gstrtph265pay.c @@ -27,6 +27,7 @@ #include #include +#include /* Included to not duplicate gst_rtp_h265_add_vps_sps_pps () */ #include "gstrtph265depay.h" @@ -1027,6 +1028,8 @@ gst_rtp_h265_pay_payload_nal (GstRTPBasePayload * basepayload, GST_BUFFER_DTS (outbuf) = dts; /* insert payload memory block */ + gst_rtp_copy_meta (GST_ELEMENT_CAST (rtph265pay), outbuf, paybuf, + g_quark_from_static_string (GST_META_TAG_VIDEO_STR)); outbuf = gst_buffer_append (outbuf, paybuf); list = gst_buffer_list_new (); @@ -1094,14 +1097,13 @@ gst_rtp_h265_pay_payload_nal (GstRTPBasePayload * basepayload, gst_rtp_buffer_unmap (&rtp); /* insert payload memory block */ - gst_buffer_append (outbuf, - gst_buffer_copy_region (paybuf, GST_BUFFER_COPY_MEMORY, pos, - limitedSize)); - + gst_rtp_copy_meta (GST_ELEMENT_CAST (rtph265pay), outbuf, paybuf, + g_quark_from_static_string (GST_META_TAG_VIDEO_STR)); + gst_buffer_copy_into (outbuf, paybuf, GST_BUFFER_COPY_MEMORY, pos, + limitedSize); /* add the buffer to the buffer list */ gst_buffer_list_add (list, outbuf); - size -= limitedSize; pos += limitedSize; ii++; @@ -1210,7 +1212,7 @@ gst_rtp_h265_pay_handle_buffer (GstRTPBasePayload * basepayload, end_of_au = TRUE; } - paybuf = gst_buffer_copy_region (buffer, GST_BUFFER_COPY_MEMORY, offset, + paybuf = gst_buffer_copy_region (buffer, GST_BUFFER_COPY_ALL, offset, nal_len); ret = From 698e5bbfb59f4b88da05aa32b992607a8dd5b9ea Mon Sep 17 00:00:00 2001 From: Luis de Bethencourt Date: Sat, 15 Aug 2015 16:22:20 +0100 Subject: [PATCH 21/26] rtph265depay: make sure we call handle_nal for each NAL Call handle_nal for each NAL in the STAP-A RTP packet. This makes sure we correctly extract the SPS and PPS. https://bugzilla.gnome.org/show_bug.cgi?id=730999 --- gst/rtp/gstrtph265depay.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/gst/rtp/gstrtph265depay.c b/gst/rtp/gstrtph265depay.c index 9718d5831a..bfc2e13a09 100644 --- a/gst/rtp/gstrtph265depay.c +++ b/gst/rtp/gstrtph265depay.c @@ -1311,7 +1311,11 @@ gst_rtp_h265_depay_process (GstRTPBaseDepayload * depayload, GstRTPBuffer * rtp) gst_rtp_copy_meta (GST_ELEMENT_CAST (rtph265depay), outbuf, buf, g_quark_from_static_string (GST_META_TAG_VIDEO_STR)); - gst_adapter_push (rtph265depay->adapter, outbuf); + outbuf = + gst_rtp_h265_depay_handle_nal (rtph265depay, outbuf, timestamp, + marker); + if (outbuf) + gst_adapter_push (rtph265depay->adapter, outbuf); payload += nalu_size; payload_len -= nalu_size; From 64ca3b26d97482a9303305185a86782813631b26 Mon Sep 17 00:00:00 2001 From: Luis de Bethencourt Date: Fri, 15 Jan 2016 15:19:41 +0000 Subject: [PATCH 22/26] rtph265pay: change config-interval property type from uint to int This way we can use -1 as special value, which is nicer than MAXUINT. https://bugzilla.gnome.org/show_bug.cgi?id=757892 --- gst/rtp/gstrtph265pay.c | 8 ++++---- gst/rtp/gstrtph265pay.h | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/gst/rtp/gstrtph265pay.c b/gst/rtp/gstrtph265pay.c index b17b091618..d72feb0f61 100644 --- a/gst/rtp/gstrtph265pay.c +++ b/gst/rtp/gstrtph265pay.c @@ -159,11 +159,11 @@ gst_rtp_h265_pay_class_init (GstRtpH265PayClass * klass) g_object_class_install_property (G_OBJECT_CLASS (klass), PROP_CONFIG_INTERVAL, - g_param_spec_uint ("config-interval", + g_param_spec_int ("config-interval", "VPS SPS PPS Send Interval", "Send VPS, SPS and PPS Insertion Interval in seconds (sprop parameter sets " "will be multiplexed in the data stream when detected.) (0 = disabled)", - 0, 3600, DEFAULT_CONFIG_INTERVAL, + -1, 3600, DEFAULT_CONFIG_INTERVAL, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS) ); @@ -1466,7 +1466,7 @@ gst_rtp_h265_pay_set_property (GObject * object, guint prop_id, rtph265pay->update_caps = TRUE; break; case PROP_CONFIG_INTERVAL: - rtph265pay->vps_sps_pps_interval = g_value_get_uint (value); + rtph265pay->vps_sps_pps_interval = g_value_get_int (value); break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); @@ -1487,7 +1487,7 @@ gst_rtp_h265_pay_get_property (GObject * object, guint prop_id, g_value_set_string (value, rtph265pay->sprop_parameter_sets); break; case PROP_CONFIG_INTERVAL: - g_value_set_uint (value, rtph265pay->vps_sps_pps_interval); + g_value_set_int (value, rtph265pay->vps_sps_pps_interval); break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); diff --git a/gst/rtp/gstrtph265pay.h b/gst/rtp/gstrtph265pay.h index 8382c5ebef..a3fa692ba4 100644 --- a/gst/rtp/gstrtph265pay.h +++ b/gst/rtp/gstrtph265pay.h @@ -64,7 +64,7 @@ struct _GstRtpH265Pay GstAdapter *adapter; - guint vps_sps_pps_interval; + gint vps_sps_pps_interval; gboolean send_vps_sps_pps; GstClockTime last_vps_sps_pps; }; From 4ee6c17edbac9d65cdba600b172a593b4b40849d Mon Sep 17 00:00:00 2001 From: Luis de Bethencourt Date: Fri, 15 Jan 2016 15:56:59 +0000 Subject: [PATCH 23/26] rtph265pay: add "send VPS/SPS/PPS with every key frame" mode It's not enough to have timeout or event based VPS/SPS/PPS information sent in RTP packets. There are some scenarios when key frames may appear more frequently than once a second, in which case the minimum timeout for "config-interval" of 1 second for sending VPS/SPS/PPS isn't enough. It might also be desirable in general to make sure the VPS/SPS/PPS is available with every keyframe (packet loss aside), so receivers can actually pick up decoding immediately from the first keyframe if VPS/SPS/PPS is not signaled out of band. This commit adds the possibility to send VPS/SPS/PPS with every key frame. This mode can be enabled by setting "config-interval" property to -1. In this case the payloader will add VPS, SPS and PPS before every key (IDR) frame. https://bugzilla.gnome.org/show_bug.cgi?id=757892 --- gst/rtp/gstrtph265pay.c | 78 ++++++++++++++++++++++------------------- 1 file changed, 42 insertions(+), 36 deletions(-) diff --git a/gst/rtp/gstrtph265pay.c b/gst/rtp/gstrtph265pay.c index d72feb0f61..db8800bc61 100644 --- a/gst/rtp/gstrtph265pay.c +++ b/gst/rtp/gstrtph265pay.c @@ -162,7 +162,8 @@ gst_rtp_h265_pay_class_init (GstRtpH265PayClass * klass) g_param_spec_int ("config-interval", "VPS SPS PPS Send Interval", "Send VPS, SPS and PPS Insertion Interval in seconds (sprop parameter sets " - "will be multiplexed in the data stream when detected.) (0 = disabled)", + "will be multiplexed in the data stream when detected.) " + "(0 = disabled, -1 = send with every IDR frame)", -1, 3600, DEFAULT_CONFIG_INTERVAL, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS) ); @@ -946,47 +947,52 @@ gst_rtp_h265_pay_payload_nal (GstRTPBasePayload * basepayload, send_vps_sps_pps = FALSE; /* check if we need to emit an VPS/SPS/PPS now */ - if (((nalType == GST_H265_NAL_SLICE_TRAIL_N) - || (nalType == GST_H265_NAL_SLICE_TRAIL_R) - || (nalType == GST_H265_NAL_SLICE_TSA_N) - || (nalType == GST_H265_NAL_SLICE_TSA_R) - || (nalType == GST_H265_NAL_SLICE_STSA_N) - || (nalType == GST_H265_NAL_SLICE_STSA_R) - || (nalType == GST_H265_NAL_SLICE_RASL_N) - || (nalType == GST_H265_NAL_SLICE_RASL_R) - || (nalType == GST_H265_NAL_SLICE_BLA_W_LP) - || (nalType == GST_H265_NAL_SLICE_BLA_W_RADL) - || (nalType == GST_H265_NAL_SLICE_BLA_N_LP) - || (nalType == GST_H265_NAL_SLICE_IDR_W_RADL) - || (nalType == GST_H265_NAL_SLICE_IDR_N_LP) - || (nalType == GST_H265_NAL_SLICE_CRA_NUT)) - && rtph265pay->vps_sps_pps_interval > 0) { + if ((nalType == GST_H265_NAL_SLICE_TRAIL_N) + || (nalType == GST_H265_NAL_SLICE_TRAIL_R) + || (nalType == GST_H265_NAL_SLICE_TSA_N) + || (nalType == GST_H265_NAL_SLICE_TSA_R) + || (nalType == GST_H265_NAL_SLICE_STSA_N) + || (nalType == GST_H265_NAL_SLICE_STSA_R) + || (nalType == GST_H265_NAL_SLICE_RASL_N) + || (nalType == GST_H265_NAL_SLICE_RASL_R) + || (nalType == GST_H265_NAL_SLICE_BLA_W_LP) + || (nalType == GST_H265_NAL_SLICE_BLA_W_RADL) + || (nalType == GST_H265_NAL_SLICE_BLA_N_LP) + || (nalType == GST_H265_NAL_SLICE_IDR_W_RADL) + || (nalType == GST_H265_NAL_SLICE_IDR_N_LP) + || (nalType == GST_H265_NAL_SLICE_CRA_NUT)) { + if (rtph265pay->vps_sps_pps_interval > 0) { + if (rtph265pay->last_vps_sps_pps != -1) { + guint64 diff; - if (rtph265pay->last_vps_sps_pps != -1) { - guint64 diff; + GST_LOG_OBJECT (rtph265pay, + "now %" GST_TIME_FORMAT ", last VPS/SPS/PPS %" GST_TIME_FORMAT, + GST_TIME_ARGS (pts), GST_TIME_ARGS (rtph265pay->last_vps_sps_pps)); - GST_LOG_OBJECT (rtph265pay, - "now %" GST_TIME_FORMAT ", last VPS/SPS/PPS %" GST_TIME_FORMAT, - GST_TIME_ARGS (pts), GST_TIME_ARGS (rtph265pay->last_vps_sps_pps)); + /* calculate diff between last SPS/PPS in milliseconds */ + if (pts > rtph265pay->last_vps_sps_pps) + diff = pts - rtph265pay->last_vps_sps_pps; + else + diff = 0; - /* calculate diff between last SPS/PPS in milliseconds */ - if (pts > rtph265pay->last_vps_sps_pps) - diff = pts - rtph265pay->last_vps_sps_pps; - else - diff = 0; + GST_DEBUG_OBJECT (rtph265pay, + "interval since last VPS/SPS/PPS %" GST_TIME_FORMAT, + GST_TIME_ARGS (diff)); - GST_DEBUG_OBJECT (rtph265pay, - "interval since last VPS/SPS/PPS %" GST_TIME_FORMAT, - GST_TIME_ARGS (diff)); - - /* bigger than interval, queue SPS/PPS */ - if (GST_TIME_AS_SECONDS (diff) >= rtph265pay->vps_sps_pps_interval) { - GST_DEBUG_OBJECT (rtph265pay, "time to send VPS/SPS/PPS"); + /* bigger than interval, queue SPS/PPS */ + if (GST_TIME_AS_SECONDS (diff) >= rtph265pay->vps_sps_pps_interval) { + GST_DEBUG_OBJECT (rtph265pay, "time to send VPS/SPS/PPS"); + send_vps_sps_pps = TRUE; + } + } else { + /* no known previous SPS/PPS time, send now */ + GST_DEBUG_OBJECT (rtph265pay, "no previous VPS/SPS/PPS time, send now"); send_vps_sps_pps = TRUE; } - } else { - /* no known previous SPS/PPS time, send now */ - GST_DEBUG_OBJECT (rtph265pay, "no previous VPS/SPS/PPS time, send now"); + } else if (rtph265pay->vps_sps_pps_interval == -1) { + GST_DEBUG_OBJECT (rtph265pay, + "sending VPS/SPS/PPS before current IDR frame"); + /* send VPS/SPS/PPS before every IDR frame */ send_vps_sps_pps = TRUE; } } From 983e30f65880876d1fd8ae1f38cc1d28d9feb003 Mon Sep 17 00:00:00 2001 From: Luis de Bethencourt Date: Fri, 15 Jan 2016 16:10:02 +0000 Subject: [PATCH 24/26] rtph265depay: fix unneeded sub-buffer creation We create a sub-buffer just to copy over its metas and then throw it away immediately, just use the original input buffer directly. --- gst/rtp/gstrtph265depay.c | 24 ++++++++---------------- 1 file changed, 8 insertions(+), 16 deletions(-) diff --git a/gst/rtp/gstrtph265depay.c b/gst/rtp/gstrtph265depay.c index bfc2e13a09..feebcf0968 100644 --- a/gst/rtp/gstrtph265depay.c +++ b/gst/rtp/gstrtph265depay.c @@ -1168,7 +1168,6 @@ static GstBuffer * gst_rtp_h265_depay_process (GstRTPBaseDepayload * depayload, GstRTPBuffer * rtp) { GstRtpH265Depay *rtph265depay; - GstBuffer *buf; GstBuffer *outbuf = NULL; guint8 nal_unit_type; @@ -1200,7 +1199,6 @@ gst_rtp_h265_depay_process (GstRTPBaseDepayload * depayload, GstRTPBuffer * rtp) payload_len = gst_rtp_buffer_get_payload_len (rtp); payload = gst_rtp_buffer_get_payload (rtp); - buf = gst_rtp_buffer_get_payload_buffer (rtp); marker = gst_rtp_buffer_get_marker (rtp); GST_DEBUG_OBJECT (rtph265depay, "receiving %d bytes", payload_len); @@ -1308,8 +1306,8 @@ gst_rtp_h265_depay_process (GstRTPBaseDepayload * depayload, GstRTPBuffer * rtp) memcpy (map.data + sizeof (sync_bytes), payload, nalu_size); gst_buffer_unmap (outbuf, &map); - gst_rtp_copy_meta (GST_ELEMENT_CAST (rtph265depay), outbuf, buf, - g_quark_from_static_string (GST_META_TAG_VIDEO_STR)); + gst_rtp_copy_meta (GST_ELEMENT_CAST (rtph265depay), outbuf, + rtp->buffer, g_quark_from_static_string (GST_META_TAG_VIDEO_STR)); outbuf = gst_rtp_h265_depay_handle_nal (rtph265depay, outbuf, timestamp, @@ -1409,8 +1407,8 @@ gst_rtp_h265_depay_process (GstRTPBaseDepayload * depayload, GstRTPBuffer * rtp) map.data[sizeof (sync_bytes) + 1] = nal_header & 0xff; gst_buffer_unmap (outbuf, &map); - gst_rtp_copy_meta (GST_ELEMENT_CAST (rtph265depay), outbuf, buf, - g_quark_from_static_string (GST_META_TAG_VIDEO_STR)); + gst_rtp_copy_meta (GST_ELEMENT_CAST (rtph265depay), outbuf, + rtp->buffer, g_quark_from_static_string (GST_META_TAG_VIDEO_STR)); GST_DEBUG_OBJECT (rtph265depay, "queueing %d bytes", outsize); @@ -1429,8 +1427,8 @@ gst_rtp_h265_depay_process (GstRTPBaseDepayload * depayload, GstRTPBuffer * rtp) outbuf = gst_buffer_new_and_alloc (outsize); gst_buffer_fill (outbuf, 0, payload, outsize); - gst_rtp_copy_meta (GST_ELEMENT_CAST (rtph265depay), outbuf, buf, - g_quark_from_static_string (GST_META_TAG_VIDEO_STR)); + gst_rtp_copy_meta (GST_ELEMENT_CAST (rtph265depay), outbuf, + rtp->buffer, g_quark_from_static_string (GST_META_TAG_VIDEO_STR)); GST_DEBUG_OBJECT (rtph265depay, "queueing %d bytes", outsize); @@ -1475,8 +1473,8 @@ gst_rtp_h265_depay_process (GstRTPBaseDepayload * depayload, GstRTPBuffer * rtp) memcpy (map.data + sizeof (sync_bytes), payload, nalu_size); gst_buffer_unmap (outbuf, &map); - gst_rtp_copy_meta (GST_ELEMENT_CAST (rtph265depay), outbuf, buf, - g_quark_from_static_string (GST_META_TAG_VIDEO_STR)); + gst_rtp_copy_meta (GST_ELEMENT_CAST (rtph265depay), outbuf, + rtp->buffer, g_quark_from_static_string (GST_META_TAG_VIDEO_STR)); outbuf = gst_rtp_h265_depay_handle_nal (rtph265depay, outbuf, timestamp, marker); @@ -1485,21 +1483,17 @@ gst_rtp_h265_depay_process (GstRTPBaseDepayload * depayload, GstRTPBuffer * rtp) } } - gst_buffer_unref (buf); - return outbuf; /* ERRORS */ empty_packet: { GST_DEBUG_OBJECT (rtph265depay, "empty packet"); - gst_buffer_unref (buf); return NULL; } waiting_start: { GST_DEBUG_OBJECT (rtph265depay, "waiting for start"); - gst_buffer_unref (buf); return NULL; } #if 0 @@ -1507,7 +1501,6 @@ not_implemented_donl_present: { GST_ELEMENT_ERROR (rtph265depay, STREAM, FORMAT, (NULL), ("DONL field present not supported yet")); - gst_buffer_unref (buf); return NULL; } #endif @@ -1515,7 +1508,6 @@ not_implemented: { GST_ELEMENT_ERROR (rtph265depay, STREAM, FORMAT, (NULL), ("NAL unit type %d not supported yet", nal_unit_type)); - gst_buffer_unref (buf); return NULL; } } From 403ac009fa9e5f28351570b186a3fe2ab4cef033 Mon Sep 17 00:00:00 2001 From: Luis de Bethencourt Date: Fri, 5 Feb 2016 13:56:34 +0000 Subject: [PATCH 25/26] rtph265depay: fix termination of access unit Only consider the access unit complete when the next-occurring VCL NAL unit has the first bit after its NAL unit header equal to 1. --- gst/rtp/gstrtph265depay.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/gst/rtp/gstrtph265depay.c b/gst/rtp/gstrtph265depay.c index feebcf0968..b7e07e19d4 100644 --- a/gst/rtp/gstrtph265depay.c +++ b/gst/rtp/gstrtph265depay.c @@ -1049,14 +1049,12 @@ gst_rtp_h265_depay_handle_nal (GstRtpH265Depay * rtph265depay, GstBuffer * nal, /* marker bit isn't mandatory so in the following code we try to detect * an AU boundary (see H.265 spec section 7.4.2.4.4) */ if (!marker) { - if (NAL_TYPE_IS_CODED_SLICE_SEGMENT (nal_type)) { /* A NAL unit (X) ends an access unit if the next-occurring VCL NAL unit (Y) has the high-order bit of the first byte after its NAL unit header equal to 1 */ start = TRUE; if (((map.data[6] >> 7) & 0x01) == 1) { complete = TRUE; } - complete = TRUE; } else if ((nal_type >= 32 && nal_type <= 35) || nal_type == 39 || (nal_type >= 41 && nal_type <= 44) || (nal_type >= 48 && nal_type <= 55)) { From 139108c83a5822fd855750872520c3cfdc5fec48 Mon Sep 17 00:00:00 2001 From: Luis de Bethencourt Date: Fri, 5 Feb 2016 15:34:51 +0000 Subject: [PATCH 26/26] gstrtph265depay: keep consistency with rtph264depay Use gst_rtp_drop_meta() and the same function prototype for gst_rtp_copy_meta() to keep consistency with the RTP elements in gst-plugins-good --- gst/rtp/gstrtph265depay.c | 31 +++++++++++++++++++++---------- gst/rtp/gstrtph265depay.h | 6 ++++++ 2 files changed, 27 insertions(+), 10 deletions(-) diff --git a/gst/rtp/gstrtph265depay.c b/gst/rtp/gstrtph265depay.c index b7e07e19d4..64dcf12cef 100644 --- a/gst/rtp/gstrtph265depay.c +++ b/gst/rtp/gstrtph265depay.c @@ -945,12 +945,12 @@ foreach_metadata_copy (GstBuffer * inbuf, GstMeta ** meta, gpointer user_data) CopyMetaData *data = user_data; GstElement *element = data->element; GstBuffer *outbuf = data->outbuf; + GQuark copy_tag = data->copy_tag; const GstMetaInfo *info = (*meta)->info; const gchar *const *tags = gst_meta_api_type_get_tags (info->api); - if (!tags || (g_strv_length ((gchar **) tags) == 1 - && gst_meta_api_type_has_tag (info->api, - g_quark_from_string (GST_META_TAG_VIDEO_STR)))) { + if (!tags || (copy_tag != 0 && g_strv_length ((gchar **) tags) == 1 + && gst_meta_api_type_has_tag (info->api, copy_tag))) { GstMetaTransformCopy copy_data = { FALSE, 0, -1 }; GST_DEBUG_OBJECT (element, "copy metadata %s", g_type_name (info->api)); /* simply copy then */ @@ -977,22 +977,32 @@ gst_rtp_copy_meta (GstElement * element, GstBuffer * outbuf, GstBuffer * inbuf, static gboolean foreach_metadata_drop (GstBuffer * inbuf, GstMeta ** meta, gpointer user_data) { - GstRtpH265Depay *depay = user_data; + DropMetaData *data = user_data; + GstElement *element = data->element; + GQuark keep_tag = data->keep_tag; const GstMetaInfo *info = (*meta)->info; const gchar *const *tags = gst_meta_api_type_get_tags (info->api); - if (!tags || (g_strv_length ((gchar **) tags) == 1 - && gst_meta_api_type_has_tag (info->api, - g_quark_from_string (GST_META_TAG_VIDEO_STR)))) { - GST_DEBUG_OBJECT (depay, "keeping metadata %s", g_type_name (info->api)); + if (!tags || (keep_tag != 0 && g_strv_length ((gchar **) tags) == 1 + && gst_meta_api_type_has_tag (info->api, keep_tag))) { + GST_DEBUG_OBJECT (element, "keeping metadata %s", g_type_name (info->api)); } else { - GST_DEBUG_OBJECT (depay, "dropping metadata %s", g_type_name (info->api)); + GST_DEBUG_OBJECT (element, "dropping metadata %s", g_type_name (info->api)); *meta = NULL; } return TRUE; } +/* TODO: Should probably make keep_tag an array at some point */ +static void +gst_rtp_drop_meta (GstElement * element, GstBuffer * buf, GQuark keep_tag) +{ + DropMetaData data = { element, keep_tag }; + + gst_buffer_foreach_meta (buf, foreach_metadata_drop, &data); +} + static GstBuffer * gst_rtp_h265_depay_handle_nal (GstRtpH265Depay * rtph265depay, GstBuffer * nal, GstClockTime in_timestamp, gboolean marker) @@ -1099,7 +1109,8 @@ gst_rtp_h265_depay_handle_nal (GstRtpH265Depay * rtph265depay, GstBuffer * nal, } outbuf = gst_buffer_make_writable (outbuf); - gst_buffer_foreach_meta (outbuf, foreach_metadata_drop, depayload); + gst_rtp_drop_meta (GST_ELEMENT_CAST (rtph265depay), outbuf, + g_quark_from_static_string (GST_META_TAG_VIDEO_STR)); GST_BUFFER_PTS (outbuf) = out_timestamp; diff --git a/gst/rtp/gstrtph265depay.h b/gst/rtp/gstrtph265depay.h index 9f50ac60e5..dba837d544 100644 --- a/gst/rtp/gstrtph265depay.h +++ b/gst/rtp/gstrtph265depay.h @@ -94,6 +94,12 @@ typedef struct GQuark copy_tag; } CopyMetaData; +typedef struct +{ + GstElement *element; + GQuark keep_tag; +} DropMetaData; + void gst_rtp_copy_meta (GstElement * element, GstBuffer *outbuf, GstBuffer *inbuf, GQuark copy_tag); GType gst_rtp_h265_depay_get_type (void);