From f577b52e5d6b7c34463817d351d82f5bd646913e Mon Sep 17 00:00:00 2001 From: Alessandro Decina Date: Fri, 30 Jan 2015 00:28:18 +1100 Subject: [PATCH] applemedia: rework GL texture sharing Use YUV instead of RGB textures, then convert using the new apple specific shader in GstGLColorConvert. Also use GLMemory directly instead of using the GL upload meta, avoiding an extra texture copy we used to have before. --- sys/applemedia/avfvideosrc.m | 17 +++-- sys/applemedia/corevideotexturecache.h | 7 ++ sys/applemedia/corevideotexturecache.m | 97 ++++++++++++++++++-------- sys/applemedia/vtdec.c | 33 +++++---- 4 files changed, 101 insertions(+), 53 deletions(-) diff --git a/sys/applemedia/avfvideosrc.m b/sys/applemedia/avfvideosrc.m index 72d9586997..134316af2c 100644 --- a/sys/applemedia/avfvideosrc.m +++ b/sys/applemedia/avfvideosrc.m @@ -56,7 +56,7 @@ static GstStaticPadTemplate src_template = GST_STATIC_PAD_TEMPLATE ("src", "height = " GST_VIDEO_SIZE_RANGE "; " GST_VIDEO_CAPS_MAKE_WITH_FEATURES - (GST_CAPS_FEATURE_META_GST_VIDEO_GL_TEXTURE_UPLOAD_META, + (GST_CAPS_FEATURE_MEMORY_GL_MEMORY, "RGBA") "; " )); @@ -398,7 +398,7 @@ didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer if (gst_format == GST_VIDEO_FORMAT_BGRA) { GstCaps *rgba_caps = GST_AVF_CAPS_NEW (GST_VIDEO_FORMAT_RGBA, dimensions.width, dimensions.height, fps_n, fps_d); - gst_caps_set_features (rgba_caps, 0, gst_caps_features_new (GST_CAPS_FEATURE_META_GST_VIDEO_GL_TEXTURE_UPLOAD_META, NULL)); + gst_caps_set_features (rgba_caps, 0, gst_caps_features_new (GST_CAPS_FEATURE_MEMORY_GL_MEMORY, NULL)); gst_caps_append (result, rgba_caps); } } @@ -748,7 +748,11 @@ didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer gst_query_parse_nth_allocation_meta (query, idx, &upload_meta_params); if (gst_structure_get (upload_meta_params, "gst.gl.GstGLContext", GST_GL_TYPE_CONTEXT, &context, NULL) && context) { + GstCaps *query_caps; + gst_query_parse_allocation (query, &query_caps, NULL); textureCache = gst_core_video_texture_cache_new (context); + gst_core_video_texture_cache_set_format (textureCache, + "NV12", query_caps); gst_object_unref (context); } } @@ -875,13 +879,8 @@ didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer } CFRelease (sbuf); - if (textureCache != NULL) { - GstVideoGLTextureType texture_types[4] = {GST_VIDEO_GL_TEXTURE_TYPE_RGBA, 0}; - gst_buffer_add_video_gl_texture_upload_meta (*buf, - GST_VIDEO_GL_TEXTURE_ORIENTATION_X_NORMAL_Y_NORMAL, - 1, texture_types, - gst_core_video_texture_cache_upload, textureCache, NULL, NULL); - } + if (textureCache != NULL) + *buf = gst_core_video_texture_cache_get_gl_buffer (textureCache, *buf); GST_BUFFER_OFFSET (*buf) = offset++; GST_BUFFER_OFFSET_END (*buf) = GST_BUFFER_OFFSET (buf) + 1; diff --git a/sys/applemedia/corevideotexturecache.h b/sys/applemedia/corevideotexturecache.h index 2e880ee155..b90cbf4ab9 100644 --- a/sys/applemedia/corevideotexturecache.h +++ b/sys/applemedia/corevideotexturecache.h @@ -34,11 +34,18 @@ typedef struct _GstCoreVideoTextureCache #else CVOpenGLESTextureCacheRef cache; #endif + GstVideoInfo input_info; + GstVideoInfo output_info; + GstGLColorConvert *convert; } GstCoreVideoTextureCache; GstCoreVideoTextureCache *gst_core_video_texture_cache_new (GstGLContext * ctx); void gst_core_video_texture_cache_free (GstCoreVideoTextureCache * cache); +void gst_core_video_texture_cache_set_format (GstCoreVideoTextureCache * cache, + const gchar * input_format, GstCaps * out_caps); gboolean gst_core_video_texture_cache_upload (GstVideoGLTextureUploadMeta * meta, guint texture_id[4]); +GstBuffer * gst_core_video_texture_cache_get_gl_buffer (GstCoreVideoTextureCache * cache, + GstBuffer * cv_buffer); G_END_DECLS diff --git a/sys/applemedia/corevideotexturecache.m b/sys/applemedia/corevideotexturecache.m index 950ae64caa..e5007af66a 100644 --- a/sys/applemedia/corevideotexturecache.m +++ b/sys/applemedia/corevideotexturecache.m @@ -36,6 +36,8 @@ gst_core_video_texture_cache_new (GstGLContext * ctx) GstCoreVideoTextureCache *cache = g_new0 (GstCoreVideoTextureCache, 1); cache->ctx = gst_object_ref (ctx); + gst_video_info_init (&cache->input_info); + cache->convert = gst_gl_color_convert_new (cache->ctx); #if !HAVE_IOS CGLPixelFormatObj pixelFormat = @@ -62,30 +64,51 @@ gst_core_video_texture_cache_free (GstCoreVideoTextureCache * cache) #else /* FIXME: how do we release ->cache ? */ #endif + gst_object_unref (cache->convert); gst_object_unref (cache->ctx); g_free (cache); } -gboolean -gst_core_video_texture_cache_upload (GstVideoGLTextureUploadMeta * meta, - guint texture_id[4]) +void +gst_core_video_texture_cache_set_format (GstCoreVideoTextureCache * cache, + const gchar * input_format, GstCaps * out_caps) { - g_return_val_if_fail (meta != NULL, FALSE); + GstCaps *in_caps; + GstCapsFeatures *features; - GstCoreVideoTextureCache *cache = - (GstCoreVideoTextureCache *) meta->user_data; - const GstGLFuncs *gl = cache->ctx->gl_vtable; + g_return_if_fail (gst_caps_is_fixed (out_caps)); + + out_caps = gst_caps_copy (out_caps); + features = gst_caps_get_features (out_caps, 0); + gst_caps_features_add (features, GST_CAPS_FEATURE_MEMORY_GL_MEMORY); + gst_video_info_from_caps (&cache->output_info, out_caps); + + in_caps = gst_caps_copy (out_caps); + gst_caps_set_simple (in_caps, "format", G_TYPE_STRING, input_format, NULL); + features = gst_caps_get_features (in_caps, 0); + gst_caps_features_add (features, GST_CAPS_FEATURE_MEMORY_GL_MEMORY); + gst_video_info_from_caps (&cache->input_info, in_caps); + + gst_gl_color_convert_set_caps (cache->convert, in_caps, out_caps); + + gst_caps_unref (out_caps); + gst_caps_unref (in_caps); +} + +CFTypeRef +texture_from_buffer (GstCoreVideoTextureCache * cache, + GstBuffer * buffer, GLuint * texture_id, GLuint * texture_target) +{ #if !HAVE_IOS CVOpenGLTextureRef texture = NULL; #else CVOpenGLESTextureRef texture = NULL; #endif - GstVideoMeta *video_meta = gst_buffer_get_video_meta (meta->buffer); GstCoreMediaMeta *cm_meta = - (GstCoreMediaMeta *) gst_buffer_get_meta (meta->buffer, + (GstCoreMediaMeta *) gst_buffer_get_meta (buffer, gst_core_media_meta_api_get_type ()); GstCoreVideoMeta *cv_meta = - (GstCoreVideoMeta *) gst_buffer_get_meta (meta->buffer, + (GstCoreVideoMeta *) gst_buffer_get_meta (buffer, gst_core_video_meta_api_get_type ()); CVPixelBufferRef pixel_buf; if (cm_meta) @@ -95,29 +118,41 @@ gst_core_video_texture_cache_upload (GstVideoGLTextureUploadMeta * meta, #if !HAVE_IOS CVOpenGLTextureCacheCreateTextureFromImage (kCFAllocatorDefault, cache->cache, pixel_buf, NULL, &texture); + *texture_id = CVOpenGLTextureGetName (texture); + *texture_target = CVOpenGLTextureGetTarget (texture); + CVOpenGLTextureCacheFlush (cache->cache, 0); #else CVOpenGLESTextureCacheCreateTextureFromImage (kCFAllocatorDefault, - cache->cache, cm_meta->image_buf, NULL, GL_TEXTURE_2D, - GL_RGBA, video_meta->width, video_meta->height, GL_RGBA, GL_UNSIGNED_BYTE, - 0, &texture); + cache->cache, cm_meta->image_buf, NULL, GL_TEXTURE_2D, GL_RGBA, + GST_VIDEO_INFO_WIDTH (&cache->input_info), + GST_VIDEO_INFO_HEIGHT (&cache->input_info), + GL_RGBA, GL_UNSIGNED_BYTE, 0, &texture); + *texture_id = CVOpenGLESTextureGetName (texture); + *texture_target = CVOpenGLESTextureGetTarget (texture); + CVOpenGLESTextureCacheFlush (cache->cache, 0); #endif - GLuint fboId; - gl->GenFramebuffers (1, &fboId); - gl->BindFramebuffer (GL_FRAMEBUFFER, fboId); - gl->FramebufferTexture2D (GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, -#if !HAVE_IOS - CVOpenGLTextureGetTarget (texture), CVOpenGLTextureGetName (texture), -#else - CVOpenGLESTextureGetTarget (texture), CVOpenGLESTextureGetName (texture), -#endif - 0); - gl->BindTexture (GL_TEXTURE_2D, texture_id[0]); - gl->CopyTexImage2D (GL_TEXTURE_2D, 0, GL_RGBA8, 0, 0, video_meta->width, - video_meta->height, 0); - gl->BindTexture (GL_TEXTURE_2D, 0); - gl->BindFramebuffer (GL_FRAMEBUFFER, 0); - gl->DeleteFramebuffers (1, &fboId); - - return TRUE; + return (CFTypeRef) texture; +} + +GstBuffer * +gst_core_video_texture_cache_get_gl_buffer (GstCoreVideoTextureCache * cache, + GstBuffer * cv_buffer) +{ + const GstGLFuncs *gl; + GstBuffer *rgb_buffer; + CFTypeRef texture; + GLuint texture_id, texture_target; + GstMemory *memory; + + gl = cache->ctx->gl_vtable; + texture = texture_from_buffer (cache, cv_buffer, &texture_id, &texture_target); + memory = (GstMemory *) gst_gl_memory_wrapped_texture (cache->ctx, texture_id, texture_target, + &cache->input_info, 0, NULL, NULL, NULL); + gst_buffer_append_memory (cv_buffer, memory); + rgb_buffer = gst_gl_color_convert_perform (cache->convert, cv_buffer); + gst_buffer_unref (cv_buffer); + CFRelease (texture); + + return rgb_buffer; } diff --git a/sys/applemedia/vtdec.c b/sys/applemedia/vtdec.c index 61efe9d151..e0a93c2337 100644 --- a/sys/applemedia/vtdec.c +++ b/sys/applemedia/vtdec.c @@ -107,7 +107,7 @@ CFSTR ("EnableHardwareAcceleratedVideoDecoder"); #define VIDEO_SRC_CAPS \ GST_VIDEO_CAPS_MAKE(GST_VTDEC_VIDEO_FORMAT_STR) ";" \ GST_VIDEO_CAPS_MAKE_WITH_FEATURES \ - (GST_CAPS_FEATURE_META_GST_VIDEO_GL_TEXTURE_UPLOAD_META, \ + (GST_CAPS_FEATURE_MEMORY_GL_MEMORY, \ "RGBA") ";" G_DEFINE_TYPE_WITH_CODE (GstVtdec, gst_vtdec, GST_TYPE_VIDEO_DECODER, @@ -301,8 +301,7 @@ gst_vtdec_set_format (GstVideoDecoder * decoder, GstVideoCodecState * state) output_state->caps = gst_video_info_to_caps (&output_state->info); if (output_state->info.finfo->format == GST_VIDEO_FORMAT_RGBA) { gst_caps_set_features (output_state->caps, 0, - gst_caps_features_new - (GST_CAPS_FEATURE_META_GST_VIDEO_GL_TEXTURE_UPLOAD_META, NULL)); + gst_caps_features_new (GST_CAPS_FEATURE_MEMORY_GL_MEMORY, NULL)); } return TRUE; @@ -397,7 +396,11 @@ gst_vtdec_create_session (GstVtdec * vtdec, GstVideoFormat format) cv_format = kCVPixelFormatType_422YpCbCr8; break; case GST_VIDEO_FORMAT_RGBA: - cv_format = kCVPixelFormatType_32BGRA; +#ifdef HAVE_IOS + cv_format = kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange; +#else + cv_format = kCVPixelFormatType_422YpCbCr8; +#endif break; default: g_warn_if_reached (); @@ -679,8 +682,16 @@ gst_vtdec_push_frames_if_needed (GstVtdec * vtdec, gboolean drain, /* negotiate now so that we know whether we need to use the GL upload meta or * not */ - if (gst_pad_check_reconfigure (decoder->srcpad)) + if (gst_pad_check_reconfigure (decoder->srcpad)) { gst_video_decoder_negotiate (decoder); + if (vtdec->texture_cache) { + GstVideoCodecState *output_state = + gst_video_decoder_get_output_state (decoder); + gst_core_video_texture_cache_set_format (vtdec->texture_cache, + GST_VTDEC_VIDEO_FORMAT_STR, output_state->caps); + gst_video_codec_state_unref (output_state); + } + } if (drain) VTDecompressionSessionWaitForAsynchronousFrames (vtdec->session); @@ -691,14 +702,10 @@ gst_vtdec_push_frames_if_needed (GstVtdec * vtdec, gboolean drain, while ((g_async_queue_length (vtdec->reorder_queue) >= vtdec->reorder_queue_length) || drain || flush) { frame = (GstVideoCodecFrame *) g_async_queue_try_pop (vtdec->reorder_queue); - if (vtdec->texture_cache != NULL) { - GstVideoGLTextureType texture_types[4] = - { GST_VIDEO_GL_TEXTURE_TYPE_RGBA, 0 }; - gst_buffer_add_video_gl_texture_upload_meta (frame->output_buffer, - GST_VIDEO_GL_TEXTURE_ORIENTATION_X_NORMAL_Y_NORMAL, 1, texture_types, - gst_core_video_texture_cache_upload, vtdec->texture_cache, NULL, - NULL); - } + if (vtdec->texture_cache != NULL) + frame->output_buffer = + gst_core_video_texture_cache_get_gl_buffer (vtdec->texture_cache, + frame->output_buffer); /* we need to check this in case reorder_queue_length=0 (jpeg for * example) or we're draining/flushing