From 3a3385f35c4566083b2b6591c739287d95e07579 Mon Sep 17 00:00:00 2001 From: Daniel Almeida Date: Tue, 18 May 2021 16:20:36 -0300 Subject: [PATCH] gl: add support for AV12 AV12 is an internally conceived format that is actually the combination of NV12 and an alpha plane. This format is to add to gstreamer's webM transparency support for vp8 and vp9. To this end, two I420 streams are independently decoded simultaneously for the actual content and the alpha plane respectively and these are then combined into A420. This patch adds GL conversion support so that it is possible to convert from AV12 to RGBA for the purposes of rendering it on a display. The reverse conversion is also supplied. Part-of: --- gst-libs/gst/gl/gstglcolorconvert.c | 62 ++++++++++++++++++++++++++++- gst-libs/gst/gl/gstglcolorconvert.h | 2 +- gst-libs/gst/gl/gstglformat.c | 3 ++ gst-libs/gst/gl/gstglmemory.h | 2 +- 4 files changed, 66 insertions(+), 3 deletions(-) diff --git a/gst-libs/gst/gl/gstglcolorconvert.c b/gst-libs/gst/gl/gstglcolorconvert.c index 203eb4c9cd..cf5f159c6b 100644 --- a/gst-libs/gst/gl/gstglcolorconvert.c +++ b/gst-libs/gst/gl/gstglcolorconvert.c @@ -333,6 +333,24 @@ static const struct shader_templ templ_SEMI_PLANAR_to_RGB = GST_GL_TEXTURE_TARGET_2D }; +static const gchar templ_AV12_to_RGB_BODY[] = + "vec4 rgba;\n" + "vec4 ayuv;\n" + /* FIXME: should get the sampling right... */ + "ayuv.x=texture2D(Ytex, texcoord * tex_scale0).r;\n" + "ayuv.yz=texture2D(UVtex, texcoord * tex_scale1).rg;\n" + "ayuv.a=texture2D(Atex, texcoord * tex_scale2).r;\n" + "rgba.rgb = yuv_to_rgb (ayuv.xyz, offset, coeff1, coeff2, coeff3);\n" + "rgba.a = ayuv.a;\n" /* copy alpha as is */ + "gl_FragColor=vec4(rgba.%c,rgba.%c,rgba.%c,rgba.%c);\n"; + +static const struct shader_templ templ_AV12_to_RGB = + { NULL, + DEFAULT_UNIFORMS YUV_TO_RGB_COEFFICIENTS "uniform sampler2D Ytex, UVtex, Atex;\n", + { glsl_func_yuv_to_rgb, NULL, }, + GST_GL_TEXTURE_TARGET_2D + }; + /* RGB to NV12/NV21/NV16/NV61 conversion */ /* NV12/NV16: u, v NV21/NV61: v, u */ @@ -354,6 +372,28 @@ static const struct shader_templ templ_RGB_to_SEMI_PLANAR_YUV = GST_GL_TEXTURE_TARGET_2D }; +static const gchar templ_RGB_to_AV12_BODY[] = + "vec4 texel, uv_texel;\n" + "vec4 ayuv;\n" + "texel = texture2D(tex, texcoord).%c%c%c%c;\n" + "uv_texel = texture2D(tex, texcoord * tex_scale0 * chroma_sampling).%c%c%c%c;\n" + "ayuv.x = rgb_to_yuv (texel.rgb, offset, coeff1, coeff2, coeff3).x;\n" + "ayuv.yz = rgb_to_yuv (uv_texel.rgb, offset, coeff1, coeff2, coeff3).yz;\n" + /* copy alpha as is */ + "ayuv.a = texel.a;\n" + "gl_FragData[0] = vec4(ayuv.x, 0.0, 0.0, 1.0);\n" + "gl_FragData[1] = vec4(ayuv.y, ayuv.z, 0.0, 1.0);\n" + "gl_FragData[2] = vec4(ayuv.a, 0.0, 0.0, 1.0);\n"; + +static const struct shader_templ templ_RGB_to_AV12 = + { NULL, + DEFAULT_UNIFORMS RGB_TO_YUV_COEFFICIENTS "uniform sampler2D tex;\n" + "uniform vec2 chroma_sampling;\n", + { glsl_func_rgb_to_yuv, NULL, }, + GST_GL_TEXTURE_TARGET_2D + }; + + /* YUY2:r,g,a UYVY:a,b,r */ static const gchar templ_YUY2_UYVY_to_RGB_BODY[] = @@ -988,7 +1028,7 @@ _init_supported_formats (GstGLContext * context, gboolean output, if (!output || (!context || context->gl_vtable->DrawBuffers)) _append_value_string_list (supported_formats, "GBRA", "GBR", "RGBP", "BGRP", "Y444", "I420", "YV12", "Y42B", "Y41B", "NV12", "NV21", "NV16", "NV61", - "A420", NULL); + "A420", "AV12", NULL); /* Requires reading from a RG/LA framebuffer... */ if (!context || (USING_GLES3 (context) || USING_OPENGL (context))) @@ -1676,6 +1716,7 @@ _get_n_textures (GstVideoFormat v_format) case GST_VIDEO_FORMAT_GBR: case GST_VIDEO_FORMAT_RGBP: case GST_VIDEO_FORMAT_BGRP: + case GST_VIDEO_FORMAT_AV12: return 3; case GST_VIDEO_FORMAT_GBRA: case GST_VIDEO_FORMAT_A420: @@ -1981,6 +2022,17 @@ _YUV_to_RGB (GstGLColorConvert * convert) info->shader_tex_names[1] = "UVtex"; break; } + case GST_VIDEO_FORMAT_AV12: + { + info->templ = &templ_AV12_to_RGB; + info->frag_body = + g_strdup_printf (templ_AV12_to_RGB_BODY, + pixel_order[0], pixel_order[1], pixel_order[2], pixel_order[3]); + info->shader_tex_names[0] = "Ytex"; + info->shader_tex_names[1] = "UVtex"; + info->shader_tex_names[2] = "Atex"; + break; + } case GST_VIDEO_FORMAT_NV21: case GST_VIDEO_FORMAT_NV61: { @@ -2132,6 +2184,14 @@ _RGB_to_YUV (GstGLColorConvert * convert) info->chroma_sampling[0] = info->chroma_sampling[1] = 2.0f; } break; + case GST_VIDEO_FORMAT_AV12: + info->templ = &templ_RGB_to_AV12, + info->frag_body = g_strdup_printf (templ_RGB_to_AV12_BODY, + pixel_order[0], pixel_order[1], pixel_order[2], pixel_order[3], + pixel_order[0], pixel_order[1], pixel_order[2], pixel_order[3]); + info->out_n_textures = 3; + info->chroma_sampling[0] = info->chroma_sampling[1] = 2.0f; + break; case GST_VIDEO_FORMAT_NV21: case GST_VIDEO_FORMAT_NV61: info->templ = &templ_RGB_to_SEMI_PLANAR_YUV, diff --git a/gst-libs/gst/gl/gstglcolorconvert.h b/gst-libs/gst/gl/gstglcolorconvert.h index 039a4c3a2f..4512fe17c6 100644 --- a/gst-libs/gst/gl/gstglcolorconvert.h +++ b/gst-libs/gst/gl/gstglcolorconvert.h @@ -103,7 +103,7 @@ struct _GstGLColorConvertClass "xBGR, ARGB, ABGR, GBRA, GBR, RGBP, BGRP, Y444, I420, YV12, Y42B, " \ "Y41B, NV12, NV21, NV16, NV61, YUY2, UYVY, Y210, AYUV, " \ "VUYA, Y410, GRAY8, GRAY16_LE, GRAY16_BE, " \ - "RGB16, BGR16, ARGB64, A420 " \ + "RGB16, BGR16, ARGB64, A420, AV12" \ GST_GL_COLOR_CONVERT_EXT_FORMATS "}" /** diff --git a/gst-libs/gst/gl/gstglformat.c b/gst-libs/gst/gl/gstglformat.c index dc940018a7..d1ce3f03a9 100644 --- a/gst-libs/gst/gl/gstglformat.c +++ b/gst-libs/gst/gl/gstglformat.c @@ -194,6 +194,9 @@ gst_gl_format_from_video_info (GstGLContext * context, case GST_VIDEO_FORMAT_NV61: n_plane_components = plane == 0 ? 1 : 2; break; + case GST_VIDEO_FORMAT_AV12: + n_plane_components = (plane == 1) ? 2 : 1; + break; case GST_VIDEO_FORMAT_GRAY8: case GST_VIDEO_FORMAT_Y444: case GST_VIDEO_FORMAT_Y42B: diff --git a/gst-libs/gst/gl/gstglmemory.h b/gst-libs/gst/gl/gstglmemory.h index 33296839f6..daea4d12a7 100644 --- a/gst-libs/gst/gl/gstglmemory.h +++ b/gst-libs/gst/gl/gstglmemory.h @@ -64,7 +64,7 @@ GType gst_gl_memory_allocator_get_type(void); #define GST_GL_MEMORY_VIDEO_FORMATS_STR \ "{ RGBA, BGRA, RGBx, BGRx, ARGB, ABGR, xRGB, xBGR, GBRA, GBR, RGBP, BGRP, RGB, BGR, RGB16, BGR16, " \ "AYUV, VUYA, Y410, I420, YV12, NV12, NV21, NV16, NV61, YUY2, UYVY, Y210, Y41B, " \ - "Y42B, Y444, GRAY8, GRAY16_LE, GRAY16_BE, ARGB64, A420" \ + "Y42B, Y444, GRAY8, GRAY16_LE, GRAY16_BE, ARGB64, A420, AV12" \ GST_GL_MEMORY_VIDEO_EXT_FORMATS "}" /**