diff --git a/ext/kate/gstkatetiger.c b/ext/kate/gstkatetiger.c index 134469fe91..f618b2e107 100644 --- a/ext/kate/gstkatetiger.c +++ b/ext/kate/gstkatetiger.c @@ -3,6 +3,7 @@ * Copyright 2005 Thomas Vander Stichele * Copyright 2005 Ronald S. Bultje * Copyright 2008 Vincent Penquerc'h + * Copyright (C) <2009> Young-Ho Cha * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), @@ -125,6 +126,57 @@ enum ARG_SILENT }; +/* RGB -> YUV blitting routines taken from textoverlay, + original code from Young-Ho Cha */ + +#define COMP_Y(ret, r, g, b) \ +{ \ + ret = (int) (((19595 * r) >> 16) + ((38470 * g) >> 16) + ((7471 * b) >> 16)); \ + ret = CLAMP (ret, 0, 255); \ +} + +#define COMP_U(ret, r, g, b) \ +{ \ + ret = (int) (-((11059 * r) >> 16) - ((21709 * g) >> 16) + ((32768 * b) >> 16) + 128); \ + ret = CLAMP (ret, 0, 255); \ +} + +#define COMP_V(ret, r, g, b) \ +{ \ + ret = (int) (((32768 * r) >> 16) - ((27439 * g) >> 16) - ((5329 * b) >> 16) + 128); \ + ret = CLAMP (ret, 0, 255); \ +} + +#define BLEND(ret, alpha, v0, v1) \ +{ \ + ret = (v0 * alpha + v1 * (255 - alpha)) / 255; \ +} + +#define OVER(ret, alphaA, Ca, alphaB, Cb, alphaNew) \ +{ \ + gint _tmp; \ + _tmp = (Ca * alphaA + Cb * alphaB * (255 - alphaA) / 255) / alphaNew; \ + ret = CLAMP (_tmp, 0, 255); \ +} + +#if G_BYTE_ORDER == G_LITTLE_ENDIAN +# define TIGER_ARGB_A 3 +# define TIGER_ARGB_R 2 +# define TIGER_ARGB_G 1 +# define TIGER_ARGB_B 0 +#else +# define TIGER_ARGB_A 0 +# define TIGER_ARGB_R 1 +# define TIGER_ARGB_G 2 +# define TIGER_ARGB_B 3 +#endif + +#define TIGER_UNPREMULTIPLY(a,r,g,b) G_STMT_START { \ + b = (a > 0) ? MIN ((b * 255 + a / 2) / a, 255) : 0; \ + g = (a > 0) ? MIN ((g * 255 + a / 2) / a, 255) : 0; \ + r = (a > 0) ? MIN ((r * 255 + a / 2) / a, 255) : 0; \ +} G_STMT_END + static GstStaticPadTemplate kate_sink_factory = GST_STATIC_PAD_TEMPLATE ("subtitle_sink", GST_PAD_SINK, @@ -134,12 +186,12 @@ static GstStaticPadTemplate kate_sink_factory = #if G_BYTE_ORDER == G_LITTLE_ENDIAN #define TIGER_VIDEO_CAPS \ - GST_VIDEO_CAPS_xRGB ", endianness = (int)1234; " \ - GST_VIDEO_CAPS_BGRx ", endianness = (int)4321" + GST_VIDEO_CAPS_xRGB ";" GST_VIDEO_CAPS_BGRx ";" \ + GST_VIDEO_CAPS_YUV ("{AYUV, I420, YV12, UYVY, NV12, NV21}") #else #define TIGER_VIDEO_CAPS \ - GST_VIDEO_CAPS_BGRx ", endianness = (int)4321; " \ - GST_VIDEO_CAPS_xRGB ", endianness = (int)1234" + GST_VIDEO_CAPS_BGRx ";" GST_VIDEO_CAPS_xRGB ";" \ + GST_VIDEO_CAPS_YUV ("{AYUV, I420, YV12, UYVY, NV12, NV21}") #endif static GstStaticPadTemplate video_sink_factory = @@ -380,6 +432,9 @@ gst_kate_tiger_dispose (GObject * object) tiger->default_font_desc = NULL; } + g_free (tiger->render_buffer); + tiger->render_buffer = NULL; + g_cond_free (tiger->cond); tiger->cond = NULL; @@ -708,6 +763,7 @@ gst_kate_tiger_video_set_caps (GstPad * pad, GstCaps * caps) tiger->swap_rgb = FALSE; if (gst_video_format_parse_caps (caps, &format, &w, &h)) { + tiger->video_format = format; tiger->video_width = w; tiger->video_height = h; } @@ -732,6 +788,405 @@ gst_kate_tiger_get_time (GstKateTiger * tiger) return pos / (gdouble) GST_SECOND; } +static inline void +gst_kate_tiger_blit_1 (GstKateTiger * tiger, guchar * dest, gint xpos, + gint ypos, const guint8 * image, gint image_width, gint image_height, + guint dest_stride) +{ + gint i, j = 0; + gint x, y; + guchar r, g, b, a; + const guint8 *pimage; + guchar *py; + gint width = image_width; + gint height = image_height; + + if (xpos < 0) { + xpos = 0; + } + + if (xpos + width > tiger->video_width) { + width = tiger->video_width - xpos; + } + + if (ypos + height > tiger->video_height) { + height = tiger->video_height - ypos; + } + + dest += (ypos / 1) * dest_stride; + + for (i = 0; i < height; i++) { + pimage = image + 4 * (i * image_width); + py = dest + i * dest_stride + xpos; + for (j = 0; j < width; j++) { + b = pimage[TIGER_ARGB_B]; + g = pimage[TIGER_ARGB_G]; + r = pimage[TIGER_ARGB_R]; + a = pimage[TIGER_ARGB_A]; + TIGER_UNPREMULTIPLY (a, r, g, b); + + pimage += 4; + if (a == 0) { + py++; + continue; + } + COMP_Y (y, r, g, b); + x = *py; + BLEND (*py++, a, y, x); + } + } +} + +static inline void +gst_kate_tiger_blit_sub2x2cbcr (GstKateTiger * tiger, + guchar * destcb, guchar * destcr, gint xpos, gint ypos, + const guint8 * image, gint image_width, gint image_height, + guint destcb_stride, guint destcr_stride, guint pix_stride) +{ + gint i, j; + gint x, cb, cr; + gushort r, g, b, a; + gushort r1, g1, b1, a1; + const guint8 *pimage1, *pimage2; + guchar *pcb, *pcr; + gint width = image_width - 2; + gint height = image_height - 2; + + xpos *= pix_stride; + + if (xpos < 0) { + xpos = 0; + } + + if (xpos + width > tiger->video_width) { + width = tiger->video_width - xpos; + } + + if (ypos + height > tiger->video_height) { + height = tiger->video_height - ypos; + } + + destcb += (ypos / 2) * destcb_stride; + destcr += (ypos / 2) * destcr_stride; + + for (i = 0; i < height; i += 2) { + pimage1 = image + 4 * (i * image_width); + pimage2 = pimage1 + 4 * image_width; + pcb = destcb + (i / 2) * destcb_stride + xpos / 2; + pcr = destcr + (i / 2) * destcr_stride + xpos / 2; + for (j = 0; j < width; j += 2) { + b = pimage1[TIGER_ARGB_B]; + g = pimage1[TIGER_ARGB_G]; + r = pimage1[TIGER_ARGB_R]; + a = pimage1[TIGER_ARGB_A]; + TIGER_UNPREMULTIPLY (a, r, g, b); + pimage1 += 4; + + b1 = pimage1[TIGER_ARGB_B]; + g1 = pimage1[TIGER_ARGB_G]; + r1 = pimage1[TIGER_ARGB_R]; + a1 = pimage1[TIGER_ARGB_A]; + TIGER_UNPREMULTIPLY (a1, r1, g1, b1); + b += b1; + g += g1; + r += r1; + a += a1; + pimage1 += 4; + + b1 = pimage2[TIGER_ARGB_B]; + g1 = pimage2[TIGER_ARGB_G]; + r1 = pimage2[TIGER_ARGB_R]; + a1 = pimage2[TIGER_ARGB_A]; + TIGER_UNPREMULTIPLY (a1, r1, g1, b1); + b += b1; + g += g1; + r += r1; + a += a1; + pimage2 += 4; + + /* + 2 for rounding */ + b1 = pimage2[TIGER_ARGB_B]; + g1 = pimage2[TIGER_ARGB_G]; + r1 = pimage2[TIGER_ARGB_R]; + a1 = pimage2[TIGER_ARGB_A]; + TIGER_UNPREMULTIPLY (a1, r1, g1, b1); + b += b1 + 2; + g += g1 + 2; + r += r1 + 2; + a += a1 + 2; + pimage2 += 4; + + b /= 4; + g /= 4; + r /= 4; + a /= 4; + + if (a == 0) { + pcb += pix_stride; + pcr += pix_stride; + continue; + } + COMP_U (cb, r, g, b); + COMP_V (cr, r, g, b); + + x = *pcb; + BLEND (*pcb, a, cb, x); + x = *pcr; + BLEND (*pcr, a, cr, x); + + pcb += pix_stride; + pcr += pix_stride; + } + } +} + +/* FIXME: + * - use proper strides and offset for I420 + */ + +static inline void +gst_kate_tiger_blit_NV12_NV21 (GstKateTiger * tiger, + guint8 * yuv_pixels, gint xpos, gint ypos, const guint8 * image, + gint image_width, gint image_height) +{ + int y_stride, uv_stride; + int u_offset, v_offset; + int h, w; + + /* because U/V is 2x2 subsampled, we need to round, either up or down, + * to a boundary of integer number of U/V pixels: + */ + xpos = GST_ROUND_UP_2 (xpos); + ypos = GST_ROUND_UP_2 (ypos); + + w = tiger->video_width; + h = tiger->video_height; + + y_stride = gst_video_format_get_row_stride (tiger->video_format, 0, w); + uv_stride = gst_video_format_get_row_stride (tiger->video_format, 1, w); + u_offset = + gst_video_format_get_component_offset (tiger->video_format, 1, w, h); + v_offset = + gst_video_format_get_component_offset (tiger->video_format, 2, w, h); + + gst_kate_tiger_blit_1 (tiger, yuv_pixels, xpos, ypos, image, image_width, + image_height, y_stride); + gst_kate_tiger_blit_sub2x2cbcr (tiger, yuv_pixels + u_offset, + yuv_pixels + v_offset, xpos, ypos, image, image_width, image_height, + uv_stride, uv_stride, 2); +} + +static inline void +gst_kate_tiger_blit_I420_YV12 (GstKateTiger * tiger, + guint8 * yuv_pixels, gint xpos, gint ypos, const guint8 * image, + gint image_width, gint image_height) +{ + int y_stride, u_stride, v_stride; + int u_offset, v_offset; + int h, w; + + /* because U/V is 2x2 subsampled, we need to round, either up or down, + * to a boundary of integer number of U/V pixels: + */ + xpos = GST_ROUND_UP_2 (xpos); + ypos = GST_ROUND_UP_2 (ypos); + + w = tiger->video_width; + h = tiger->video_height; + + y_stride = gst_video_format_get_row_stride (tiger->video_format, 0, w); + u_stride = gst_video_format_get_row_stride (tiger->video_format, 1, w); + v_stride = gst_video_format_get_row_stride (tiger->video_format, 2, w); + u_offset = + gst_video_format_get_component_offset (tiger->video_format, 1, w, h); + v_offset = + gst_video_format_get_component_offset (tiger->video_format, 2, w, h); + + gst_kate_tiger_blit_1 (tiger, yuv_pixels, xpos, ypos, image, image_width, + image_height, y_stride); + gst_kate_tiger_blit_sub2x2cbcr (tiger, yuv_pixels + u_offset, + yuv_pixels + v_offset, xpos, ypos, image, image_width, image_height, + u_stride, v_stride, 1); +} + +static inline void +gst_kate_tiger_blit_UYVY (GstKateTiger * tiger, + guint8 * yuv_pixels, gint xpos, gint ypos, const guint8 * image, + gint image_width, gint image_height) +{ + int a0, r0, g0, b0; + int a1, r1, g1, b1; + int y0, y1, u, v; + int i, j; + int h, w; + const guint8 *pimage; + guchar *dest; + + /* because U/V is 2x horizontally subsampled, we need to round to a + * boundary of integer number of U/V pixels in x dimension: + */ + xpos = GST_ROUND_UP_2 (xpos); + + w = image_width - 2; + h = image_height - 2; + + if (xpos < 0) { + xpos = 0; + } + + if (xpos + w > tiger->video_width) { + w = tiger->video_width - xpos; + } + + if (ypos + h > tiger->video_height) { + h = tiger->video_height - ypos; + } + + for (i = 0; i < h; i++) { + pimage = image + i * image_width * 4; + dest = yuv_pixels + (i + ypos) * tiger->video_width * 2 + xpos * 2; + for (j = 0; j < w; j += 2) { + b0 = pimage[TIGER_ARGB_B]; + g0 = pimage[TIGER_ARGB_G]; + r0 = pimage[TIGER_ARGB_R]; + a0 = pimage[TIGER_ARGB_A]; + TIGER_UNPREMULTIPLY (a0, r0, g0, b0); + pimage += 4; + + b1 = pimage[TIGER_ARGB_B]; + g1 = pimage[TIGER_ARGB_G]; + r1 = pimage[TIGER_ARGB_R]; + a1 = pimage[TIGER_ARGB_A]; + TIGER_UNPREMULTIPLY (a1, r1, g1, b1); + pimage += 4; + + a0 += a1 + 2; + a0 /= 2; + if (a0 == 0) { + dest += 4; + continue; + } + + COMP_Y (y0, r0, g0, b0); + COMP_Y (y1, r1, g1, b1); + + b0 += b1 + 2; + g0 += g1 + 2; + r0 += r1 + 2; + + b0 /= 2; + g0 /= 2; + r0 /= 2; + + COMP_U (u, r0, g0, b0); + COMP_V (v, r0, g0, b0); + + BLEND (*dest, a0, u, *dest); + dest++; + BLEND (*dest, a0, y0, *dest); + dest++; + BLEND (*dest, a0, v, *dest); + dest++; + BLEND (*dest, a0, y1, *dest); + dest++; + } + } +} + +static inline void +gst_kate_tiger_blit_AYUV (GstKateTiger * tiger, + guint8 * rgb_pixels, gint xpos, gint ypos, const guint8 * image, + gint image_width, gint image_height) +{ + int a, r, g, b, a1; + int y, u, v; + int i, j; + int h, w; + const guint8 *pimage; + guchar *dest; + + w = image_width; + h = image_height; + + if (xpos < 0) { + xpos = 0; + } + + if (xpos + w > tiger->video_width) { + w = tiger->video_width - xpos; + } + + if (ypos + h > tiger->video_height) { + h = tiger->video_height - ypos; + } + + for (i = 0; i < h; i++) { + pimage = image + i * image_width * 4; + dest = rgb_pixels + (i + ypos) * 4 * tiger->video_width + xpos * 4; + for (j = 0; j < w; j++) { + a = pimage[TIGER_ARGB_A]; + b = pimage[TIGER_ARGB_B]; + g = pimage[TIGER_ARGB_G]; + r = pimage[TIGER_ARGB_R]; + + TIGER_UNPREMULTIPLY (a, r, g, b); + + // convert background to yuv + COMP_Y (y, r, g, b); + COMP_U (u, r, g, b); + COMP_V (v, r, g, b); + + // preform text "OVER" background alpha compositing + a1 = a + (dest[0] * (255 - a)) / 255 + 1; // add 1 to prevent divide by 0 + OVER (dest[1], a, y, dest[0], dest[1], a1); + OVER (dest[2], a, u, dest[0], dest[2], a1); + OVER (dest[3], a, v, dest[0], dest[3], a1); + dest[0] = a1 - 1; // remove the temporary 1 we added + + pimage += 4; + dest += 4; + } + } +} + +static void +gst_kate_tiger_blend_yuv (GstKateTiger * tiger, GstBuffer * video_frame, + const guint8 * image, gint image_width, gint image_height) +{ + gint xpos = 0, ypos = 0; + gint width, height; + + width = image_width; + height = image_height; + + switch (tiger->video_format) { + case GST_VIDEO_FORMAT_I420: + case GST_VIDEO_FORMAT_YV12: + gst_kate_tiger_blit_I420_YV12 (tiger, + GST_BUFFER_DATA (video_frame), xpos, ypos, image, image_width, + image_height); + break; + case GST_VIDEO_FORMAT_NV12: + case GST_VIDEO_FORMAT_NV21: + gst_kate_tiger_blit_NV12_NV21 (tiger, + GST_BUFFER_DATA (video_frame), xpos, ypos, image, image_width, + image_height); + break; + case GST_VIDEO_FORMAT_UYVY: + gst_kate_tiger_blit_UYVY (tiger, + GST_BUFFER_DATA (video_frame), xpos, ypos, image, image_width, + image_height); + break; + case GST_VIDEO_FORMAT_AYUV: + gst_kate_tiger_blit_AYUV (tiger, + GST_BUFFER_DATA (video_frame), xpos, ypos, image, image_width, + image_height); + break; + default: + g_assert_not_reached (); + } +} + static GstFlowReturn gst_kate_tiger_video_chain (GstPad * pad, GstBuffer * buf) { @@ -758,8 +1213,9 @@ gst_kate_tiger_video_chain (GstPad * pad, GstBuffer * buf) g_cond_broadcast (tiger->cond); } - /* Update first with a dummy buffer pointer we cannot write to. If there is nothing - to draw, we will not have to make it writeable */ + /* Update first with a dummy buffer pointer we cannot write to, but with the + right dimensions. If there is nothing to draw, we will not have to make + it writeable. */ ptr = GST_BUFFER_DATA (buf); ret = tiger_renderer_set_buffer (tiger->tr, ptr, tiger->video_width, @@ -792,7 +1248,19 @@ gst_kate_tiger_video_chain (GstPad * pad, GstBuffer * buf) } /* and setup that buffer before rendering */ - ptr = GST_BUFFER_DATA (buf); + if (gst_video_format_is_yuv (tiger->video_format)) { + guint8 *tmp = g_realloc (tiger->render_buffer, + tiger->video_width * tiger->video_height * 4); + if (!tmp) { + GST_WARNING_OBJECT (tiger, "Failed to allocate render buffer"); + goto pass; + } + tiger->render_buffer = tmp; + ptr = tiger->render_buffer; + tiger_renderer_set_surface_clear_color (tiger->tr, 1, 0.0, 0.0, 0.0, 0.0); + } else { + ptr = GST_BUFFER_DATA (buf); + } ret = tiger_renderer_set_buffer (tiger->tr, ptr, tiger->video_width, tiger->video_height, tiger->video_width * 4, tiger->swap_rgb); @@ -809,6 +1277,11 @@ gst_kate_tiger_video_chain (GstPad * pad, GstBuffer * buf) GST_LOG_OBJECT (tiger, "Tiger renderer rendered on video frame at %f", t); } + if (gst_video_format_is_yuv (tiger->video_format)) { + gst_kate_tiger_blend_yuv (tiger, buf, tiger->render_buffer, + tiger->video_width, tiger->video_height); + } + pass: GST_KATE_TIGER_MUTEX_UNLOCK (tiger); diff --git a/ext/kate/gstkatetiger.h b/ext/kate/gstkatetiger.h index 0947da4cff..f966cbf152 100644 --- a/ext/kate/gstkatetiger.h +++ b/ext/kate/gstkatetiger.h @@ -49,6 +49,7 @@ #include #include #include +#include #include "gstkateutil.h" G_BEGIN_DECLS @@ -90,9 +91,11 @@ struct _GstKateTiger guchar default_background_a; gboolean silent; + GstVideoFormat video_format; gint video_width; gint video_height; gboolean swap_rgb; + guint8 *render_buffer; GMutex *mutex; GCond *cond; diff --git a/gst/mpeg4videoparse/mpeg4parse.c b/gst/mpeg4videoparse/mpeg4parse.c index c00c85a6c2..5c8ce9574c 100644 --- a/gst/mpeg4videoparse/mpeg4parse.c +++ b/gst/mpeg4videoparse/mpeg4parse.c @@ -164,8 +164,8 @@ gst_mpeg4_params_parse_vo (MPEG4Params * params, GstBitReader * br) int n; /* Length of the time increment is the minimal number of bits needed to - * represent time_increment_resolution */ - for (n = 0; (time_increment_resolution >> n) != 0; n++); + * represent time_increment_resolution-1 */ + for (n = 0; ((time_increment_resolution - 1) >> n) != 0; n++); GET_BITS (br, n, &bits); fixed_time_increment = bits; diff --git a/gst/mpegdemux/gstmpegtsdemux.c b/gst/mpegdemux/gstmpegtsdemux.c index b3b8249e61..394d0b36d2 100644 --- a/gst/mpegdemux/gstmpegtsdemux.c +++ b/gst/mpegdemux/gstmpegtsdemux.c @@ -88,6 +88,10 @@ enum /* latency in mseconds */ #define TS_LATENCY 700 +/* threshold at which we deem PTS difference to be a discontinuity */ +#define DISCONT_THRESHOLD_AV (GST_SECOND * 2) /* 2 seconds */ +#define DISCONT_THRESHOLD_OTHER (GST_SECOND * 60 * 10) /* 10 minutes */ + enum { PROP_0, @@ -332,6 +336,9 @@ gst_mpegts_demux_init (GstMpegTSDemux * demux) demux->pcr[1] = -1; demux->cache_duration = GST_CLOCK_TIME_NONE; demux->base_pts = GST_CLOCK_TIME_NONE; + demux->in_gap = GST_CLOCK_TIME_NONE; + demux->first_buf_ts = GST_CLOCK_TIME_NONE; + demux->last_buf_ts = GST_CLOCK_TIME_NONE; } static void @@ -382,6 +389,10 @@ gst_mpegts_demux_reset (GstMpegTSDemux * demux) g_object_unref (demux->clock); demux->clock = NULL; } + + demux->in_gap = GST_CLOCK_TIME_NONE; + demux->first_buf_ts = GST_CLOCK_TIME_NONE; + demux->last_buf_ts = GST_CLOCK_TIME_NONE; } #if 0 @@ -539,6 +550,20 @@ gst_mpegts_stream_is_video (GstMpegTSStream * stream) return FALSE; } +static FORCE_INLINE gboolean +gst_mpegts_stream_is_audio (GstMpegTSStream * stream) +{ + switch (stream->stream_type) { + case ST_AUDIO_MPEG1: + case ST_AUDIO_MPEG2: + case ST_AUDIO_AAC_ADTS: + case ST_AUDIO_AAC_LOAS: + return TRUE; + } + + return FALSE; +} + static gboolean gst_mpegts_demux_is_reserved_PID (GstMpegTSDemux * demux, guint16 PID) { @@ -1026,6 +1051,110 @@ done: return ret; } +static void +gst_mpegts_demux_sync_streams (GstMpegTSDemux * demux, GstClockTime time) +{ + gint i; + + for (i = 0; i < MPEGTS_MAX_PID + 1; i++) { + GstMpegTSStream *stream = demux->streams[i]; + if (!stream) + continue; + + /* Theoretically, we should be doing this for all streams, but we're only + * doing it for non A/V streams, for which data might not be forthcoming. */ + if (stream->flags & (MPEGTS_STREAM_FLAG_IS_AUDIO | + MPEGTS_STREAM_FLAG_IS_VIDEO)) + continue; + + /* at start, lock all streams onto the first timestamp */ + if (G_UNLIKELY (stream->last_time == 0)) + stream->last_time = time; + + /* Does this stream lag? Random threshold of 2 seconds */ + if (GST_CLOCK_DIFF (stream->last_time, time) > (2 * GST_SECOND)) { + /* If the pad was not added yet, do not wait any longer for + any pad that might be waiting for data */ + if (!stream->pad && demux->pending_pads > 0) { + demux->pending_pads = 0; + gst_element_no_more_pads (GST_ELEMENT (demux)); + } + + if (stream->pad) { + GST_DEBUG_OBJECT (stream, "synchronizing stream with others by " + "advancing time from %" GST_TIME_FORMAT " to %" GST_TIME_FORMAT, + GST_TIME_ARGS (stream->last_time), GST_TIME_ARGS (time)); + stream->last_time = time; + /* advance stream time (FIXME: is this right, esp. time_pos?) */ + gst_pad_push_event (stream->pad, + gst_event_new_new_segment (TRUE, 1.0, + GST_FORMAT_TIME, stream->last_time, -1, stream->last_time)); + } + } + } +} + +/* Attempts to add all known streams. + Returns TRUE if all could be added, FALSE otherwise. + */ +static gboolean +gst_mpegts_demux_add_all_streams (GstMpegTSDemux * demux, GstClockTime pts) +{ + guint i; + GstPad *srcpad; + gboolean all_added = TRUE; + + /* When adding a stream, require either a valid base PCR, or a valid PTS */ + if (!gst_mpegts_demux_setup_base_pts (demux, pts)) { + GST_ERROR ("Can't set base pts"); + return FALSE; + } + + for (i = 0; i < MPEGTS_MAX_PID + 1; i++) { + GstMpegTSStream *stream = demux->streams[i]; + if (!stream || stream->pad) + continue; + + GST_DEBUG_OBJECT (demux, "Trying to add pad for PID 0x%04x", stream->PID); + + if (demux->current_PMT == 0) { + if (G_UNLIKELY (stream->flags & MPEGTS_STREAM_FLAG_STREAM_TYPE_UNKNOWN)) { + GST_DEBUG_OBJECT (demux, + "Stream flagged as unknown, cannot be added now"); + all_added = FALSE; + continue; + } + } + if (!gst_mpegts_demux_fill_stream (stream, stream->filter.id, + stream->stream_type)) { + GST_ERROR ("Unknown type for PID 0x%04x", stream->PID); + /* ignore */ + continue; + } + + GST_DEBUG_OBJECT (demux, + "New stream 0x%04x of type 0x%02x with caps %" GST_PTR_FORMAT, + stream->PID, stream->stream_type, GST_PAD_CAPS (stream->pad)); + + srcpad = stream->pad; + + /* activate and add */ + gst_pad_set_active (srcpad, TRUE); + gst_element_add_pad (GST_ELEMENT_CAST (demux), srcpad); + demux->need_no_more_pads = TRUE; + + stream->discont = TRUE; + + /* send new_segment */ + gst_mpegts_demux_send_new_segment (demux, stream, pts); + + /* send tags */ + gst_mpegts_demux_send_tags_for_stream (demux, stream); + + } + + return all_added; +} static GstFlowReturn gst_mpegts_demux_data_cb (GstPESFilter * filter, gboolean first, @@ -1040,23 +1169,28 @@ gst_mpegts_demux_data_cb (GstPESFilter * filter, gboolean first, demux = stream->demux; srcpad = stream->pad; - GST_DEBUG_OBJECT (demux, "got data on PID 0x%04x", stream->PID); + GST_DEBUG_OBJECT (demux, "got data on PID 0x%04x (flags %x)", stream->PID, + stream->flags); if (first && filter->pts != -1) { + gint64 discont_threshold = + ((stream->flags & (MPEGTS_STREAM_FLAG_IS_AUDIO | + MPEGTS_STREAM_FLAG_IS_VIDEO))) ? DISCONT_THRESHOLD_AV : + DISCONT_THRESHOLD_OTHER; pts = filter->pts; time = MPEGTIME_TO_GSTTIME (pts) + stream->base_time; if ((stream->last_time > 0 && stream->last_time < time && - time - stream->last_time > GST_SECOND * 60 * 10) + time - stream->last_time > discont_threshold) || (stream->last_time > time - && stream->last_time - time > GST_SECOND * 60 * 10)) { + && stream->last_time - time > discont_threshold)) { /* check first to see if we're in middle of detecting a discont in PCR. * if we are we're not sure what timestamp the buffer should have, best * to drop. */ if (stream->PMT_pid <= MPEGTS_MAX_PID && demux->streams[stream->PMT_pid] && demux->streams[demux->streams[stream->PMT_pid]->PMT.PCR_PID] - && demux->streams[demux->streams[stream->PMT_pid]->PMT. - PCR_PID]->discont_PCR) { + && demux->streams[demux->streams[stream->PMT_pid]->PMT.PCR_PID]-> + discont_PCR) { GST_WARNING_OBJECT (demux, "middle of discont, dropping"); goto bad_timestamp; } @@ -1065,8 +1199,7 @@ gst_mpegts_demux_data_cb (GstPESFilter * filter, gboolean first, stream->last_time - time > MPEGTIME_TO_GSTTIME (G_MAXUINT32)) { /* wrap around occurred */ if (stream->base_time + MPEGTIME_TO_GSTTIME ((guint64) (1) << 33) + - MPEGTIME_TO_GSTTIME (pts) > - stream->last_time + GST_SECOND * 60 * 10) { + MPEGTIME_TO_GSTTIME (pts) > stream->last_time + discont_threshold) { GST_DEBUG_OBJECT (demux, "looks like we have a corrupt packet because its pts is a lot lower than" " the previous pts but not a wraparound"); @@ -1078,8 +1211,8 @@ gst_mpegts_demux_data_cb (GstPESFilter * filter, gboolean first, */ if (stream->PMT_pid <= MPEGTS_MAX_PID && demux->streams[stream->PMT_pid] && demux->streams[demux->streams[stream->PMT_pid]->PMT.PCR_PID] - && demux->streams[demux->streams[stream->PMT_pid]->PMT. - PCR_PID]->last_PCR > 0) { + && demux->streams[demux->streams[stream->PMT_pid]->PMT.PCR_PID]-> + last_PCR > 0) { GST_DEBUG_OBJECT (demux, "timestamps wrapped before noticed in PCR"); time = MPEGTIME_TO_GSTTIME (pts) + stream->base_time + MPEGTIME_TO_GSTTIME ((guint64) (1) << 33); @@ -1100,7 +1233,7 @@ gst_mpegts_demux_data_cb (GstPESFilter * filter, gboolean first, stream->base_time > 0) { /* had a previous wrap around */ if (time - MPEGTIME_TO_GSTTIME ((guint64) (1) << 33) + - GST_SECOND * 60 * 10 < stream->last_time) { + discont_threshold < stream->last_time) { GST_DEBUG_OBJECT (demux, "looks like we have a corrupt packet because its pts is a lot higher than" " the previous pts but not because of a wraparound or pcr discont"); @@ -1144,15 +1277,58 @@ gst_mpegts_demux_data_cb (GstPESFilter * filter, gboolean first, pts = -1; } + if (G_UNLIKELY (!GST_CLOCK_TIME_IS_VALID (demux->in_gap))) { + if (GST_CLOCK_TIME_IS_VALID (demux->first_buf_ts) + && GST_CLOCK_TIME_IS_VALID (filter->pts)) { + int i; + GstClockTime pts = GST_CLOCK_TIME_NONE; + for (i = 0; i < MPEGTS_MAX_PID + 1; i++) { + GstMpegTSStream *stream = demux->streams[i]; + if (stream && (pts == GST_CLOCK_TIME_NONE || stream->last_time < pts)) { + pts = stream->last_time; + } + } + if (pts == GST_CLOCK_TIME_NONE) + pts = 0; + demux->in_gap = demux->first_buf_ts - pts; + GST_INFO_OBJECT (demux, "Setting interpolation gap to %" GST_TIME_FORMAT, + GST_TIME_ARGS (demux->in_gap)); + } else { + demux->in_gap = 0; + } + } + + if (GST_CLOCK_TIME_IS_VALID (time)) { + time += demux->in_gap; + } + GST_DEBUG_OBJECT (demux, "setting PTS to (%" G_GUINT64_FORMAT ") time: %" GST_TIME_FORMAT " on buffer %p first buffer: %d base_time: %" - GST_TIME_FORMAT, pts, GST_TIME_ARGS (time), buffer, first, - GST_TIME_ARGS (stream->base_time)); + GST_TIME_FORMAT, pts, GST_TIME_ARGS (time + demux->in_gap), + buffer, first, GST_TIME_ARGS (stream->base_time)); GST_BUFFER_TIMESTAMP (buffer) = time; /* check if we have a pad already */ + if (!demux->tried_adding_pads) { + GST_DEBUG_OBJECT (demux, "Trying to add all pads now"); + if (gst_mpegts_demux_add_all_streams (demux, pts)) { + /* We managed to add all pads, so we can signal no-more-pads safely. + If not, we'll add pads as we get data for them, and will end up + hitting decodebin2's overrun threshold (if using decodebin2) */ + GST_DEBUG_OBJECT (demux, "All pads added, we can signal no-more-pads"); + gst_element_no_more_pads (GST_ELEMENT (demux)); + } else { + GST_DEBUG_OBJECT (demux, + "All pads could not be added, we will not signal no-more-pads"); + } + demux->tried_adding_pads = TRUE; + } + + + srcpad = stream->pad; if (srcpad == NULL) { + GST_DEBUG_OBJECT (demux, "srcpad is NULL, trying to add pad"); /* When adding a stream, require either a valid base PCR, or a valid PTS */ if (!gst_mpegts_demux_setup_base_pts (demux, pts)) goto bad_timestamp; @@ -1191,7 +1367,12 @@ gst_mpegts_demux_data_cb (GstPESFilter * filter, gboolean first, /* activate and add */ gst_pad_set_active (srcpad, TRUE); gst_element_add_pad (GST_ELEMENT_CAST (demux), srcpad); - demux->need_no_more_pads = TRUE; + demux->pending_pads--; + GST_DEBUG_OBJECT (demux, + "Adding pad due to received data, decreasing pending pads to %d", + demux->pending_pads); + if (demux->pending_pads == 0) + gst_element_no_more_pads (GST_ELEMENT (demux)); stream->discont = TRUE; @@ -1202,7 +1383,8 @@ gst_mpegts_demux_data_cb (GstPESFilter * filter, gboolean first, gst_mpegts_demux_send_tags_for_stream (demux, stream); } - GST_DEBUG_OBJECT (srcpad, "pushing buffer"); + GST_DEBUG_OBJECT (srcpad, "pushing buffer ts %" GST_TIME_FORMAT, + GST_TIME_ARGS (GST_BUFFER_TIMESTAMP (buffer))); gst_buffer_set_caps (buffer, GST_PAD_CAPS (srcpad)); if (stream->discont) { GST_BUFFER_FLAG_SET (buffer, GST_BUFFER_FLAG_DISCONT); @@ -1211,6 +1393,9 @@ gst_mpegts_demux_data_cb (GstPESFilter * filter, gboolean first, ret = gst_pad_push (srcpad, buffer); ret = gst_mpegts_demux_combine_flows (demux, stream, ret); + if (GST_CLOCK_TIME_IS_VALID (time)) + gst_mpegts_demux_sync_streams (demux, time); + return ret; /* ERROR */ @@ -1416,6 +1601,8 @@ gst_mpegts_stream_parse_pmt (GstMpegTSStream * stream, g_array_free (PMT->entries, TRUE); PMT->entries = g_array_new (FALSE, TRUE, sizeof (GstMpegTSPMTEntry)); + GST_DEBUG_OBJECT (demux, "Resetting pending pads due to parsing the PMT"); + demux->pending_pads = 0; while (entries > 0) { GstMpegTSPMTEntry entry; GstMpegTSStream *ES_stream; @@ -1479,6 +1666,9 @@ gst_mpegts_stream_parse_pmt (GstMpegTSStream * stream, /* Recognise video streams based on stream_type */ if (gst_mpegts_stream_is_video (ES_stream)) ES_stream->flags |= MPEGTS_STREAM_FLAG_IS_VIDEO; + /* likewise for audio */ + if (gst_mpegts_stream_is_audio (ES_stream)) + ES_stream->flags |= MPEGTS_STREAM_FLAG_IS_AUDIO; /* set adaptor */ GST_LOG ("Initializing PES filter for PID %u", ES_stream->PID); @@ -1496,6 +1686,12 @@ gst_mpegts_stream_parse_pmt (GstMpegTSStream * stream, ES_stream->filter.gather_pes = TRUE; } } + + ++demux->pending_pads; + GST_DEBUG_OBJECT (demux, + "Setting data callback, increasing pending pads to %d", + demux->pending_pads); + gst_pes_filter_set_callbacks (&ES_stream->filter, (GstPESFilterData) gst_mpegts_demux_data_cb, (GstPESFilterResync) gst_mpegts_demux_resync_cb, ES_stream); @@ -1527,6 +1723,11 @@ gst_mpegts_stream_parse_pmt (GstMpegTSStream * stream, gst_mpegts_activate_pmt (demux, stream); } + GST_DEBUG_OBJECT (demux, "Done parsing PMT, pending pads now %d", + demux->pending_pads); + if (demux->pending_pads == 0) + gst_element_no_more_pads (GST_ELEMENT (demux)); + return TRUE; /* ERRORS */ @@ -1751,9 +1952,9 @@ gst_mpegts_demux_parse_adaptation_field (GstMpegTSStream * stream, stream->last_PCR, GST_TIME_ARGS (MPEGTIME_TO_GSTTIME (stream->last_PCR))); /* pcr has been converted into units of 90Khz ticks - * so assume discont if last pcr was > 900000 (10 second) lower */ + * so assume discont if last pcr was > 90000 (1 second) lower */ if (stream->last_PCR != -1 && - (pcr - stream->last_PCR > 900000 || pcr < stream->last_PCR)) { + (pcr - stream->last_PCR > 90000 || pcr < stream->last_PCR)) { GstClockTimeDiff base_time_difference; GST_DEBUG_OBJECT (demux, @@ -2694,6 +2895,9 @@ gst_mpegts_demux_sink_event (GstPad * pad, GstEvent * event) gst_adapter_clear (demux->adapter); gst_mpegts_demux_flush (demux, TRUE); res = gst_mpegts_demux_send_event (demux, event); + demux->in_gap = GST_CLOCK_TIME_NONE; + demux->first_buf_ts = GST_CLOCK_TIME_NONE; + demux->last_buf_ts = GST_CLOCK_TIME_NONE; break; case GST_EVENT_EOS: gst_mpegts_demux_flush (demux, FALSE); @@ -3034,7 +3238,35 @@ gst_mpegts_demux_chain (GstPad * pad, GstBuffer * buffer) gint i; guint sync_count; + if (GST_BUFFER_TIMESTAMP_IS_VALID (buffer)) { + GstClockTime timestamp = GST_BUFFER_TIMESTAMP (buffer); + GST_DEBUG_OBJECT (demux, "Got chained buffer ts %" GST_TIME_FORMAT, + GST_TIME_ARGS (timestamp)); + + /* if we did not get a buffer for a while, assume the source has dried up, + and flush any stale data */ + if (GST_CLOCK_TIME_IS_VALID (demux->last_buf_ts)) { + GstClockTimeDiff dt = timestamp - demux->last_buf_ts; + if (dt < 0 || dt > GST_SECOND / 2) { + GST_INFO_OBJECT (demux, + "Input timestamp discontinuity (%" GST_TIME_FORMAT + "), flushing stale data", GST_TIME_ARGS (dt)); + gst_mpegts_demux_flush (demux, FALSE); + } + } + demux->last_buf_ts = timestamp; + + /* lock on the first valid buffer timestamp */ + if (G_UNLIKELY (demux->first_buf_ts == GST_CLOCK_TIME_NONE)) { + demux->first_buf_ts = timestamp; + GST_DEBUG_OBJECT (demux, "First timestamp is %" GST_TIME_FORMAT, + GST_TIME_ARGS (demux->first_buf_ts)); + } + } + if (GST_BUFFER_IS_DISCONT (buffer)) { + GST_DEBUG_OBJECT (demux, + "Input buffer has DISCONT flag set, flushing data"); gst_mpegts_demux_flush (demux, FALSE); } /* first push the new buffer into the adapter */ diff --git a/gst/mpegdemux/gstmpegtsdemux.h b/gst/mpegdemux/gstmpegtsdemux.h index c3732ba1f0..935ab403f5 100644 --- a/gst/mpegdemux/gstmpegtsdemux.h +++ b/gst/mpegdemux/gstmpegtsdemux.h @@ -121,7 +121,8 @@ struct _GstMpegTSPAT { typedef enum _MpegTsStreamFlags { MPEGTS_STREAM_FLAG_STREAM_TYPE_UNKNOWN = 0x01, MPEGTS_STREAM_FLAG_PMT_VALID = 0x02, - MPEGTS_STREAM_FLAG_IS_VIDEO = 0x04 + MPEGTS_STREAM_FLAG_IS_VIDEO = 0x04, + MPEGTS_STREAM_FLAG_IS_AUDIO = 0x08 } MpegTsStreamFlags; /* Information associated to a single MPEG stream. */ @@ -223,6 +224,18 @@ struct _GstMpegTSDemux { /* Cached base_PCR in GStreamer time. */ GstClockTime base_pts; + + /* base timings on first buffer timestamp */ + GstClockTime first_buf_ts; + GstClockTime in_gap; + + /* Detect when the source stops for a while, we will resync the interpolation gap */ + GstClockTime last_buf_ts; + + /* Number of expected pads which have not been added yet */ + gint pending_pads; + + gboolean tried_adding_pads; }; struct _GstMpegTSDemuxClass {