From 5fd66383ab85b8296226a7ba81ffaa2d91b4c0e4 Mon Sep 17 00:00:00 2001 From: Filippo Argiolas Date: Tue, 27 Apr 2010 19:38:33 +0200 Subject: [PATCH] [441/906] sobel: convolve only luma Rework Sobel a little bit again making it work as the old one: 1. desaturate input texture 2. calculate horizontal convolution for x gradient and vertical convolution for y gradient at the same time (halves the number of needed texture lookups) 3. store results in a single texture (red and green channel) 4. calculate remaining convolution (same as above switching vertical and horizontal) 5. calculate length of gradient using red and green as x and y components. Optimize wherever possible, store kernels as constants in the shaders, remove unneeded uniforms. Restore invert property carefully avoiding using IF. Still not sure if "full color" convolution will be needed, glfiltersobel is to be intended as a demo filter and xray, the only effect which uses sobel only needs edge intensity. Dropping it for now. --- gst/gl/effects/gstgleffectssources.c | 89 +++++++++------- gst/gl/effects/gstgleffectssources.h | 7 +- gst/gl/gstglfiltersobel.c | 150 ++++++++++----------------- gst/gl/gstglfiltersobel.h | 1 + 4 files changed, 112 insertions(+), 135 deletions(-) diff --git a/gst/gl/effects/gstgleffectssources.c b/gst/gl/effects/gstgleffectssources.c index 27b2c749b5..b26be169b8 100644 --- a/gst/gl/effects/gstgleffectssources.c +++ b/gst/gl/effects/gstgleffectssources.c @@ -308,60 +308,79 @@ const gchar *sobel_fragment_source = " gl_FragColor = vec4(vec3(g), 1.0);" "}"; -const gchar *sobel_gradient_length_fragment_source = - "#extension GL_ARB_texture_rectangle : enable\n" - "uniform sampler2DRect gx;" - "uniform sampler2DRect gy;" - "void main () {" - " vec4 dx = texture2DRect (gx, gl_TexCoord[0].st);" - " vec4 dy = texture2DRect (gy, gl_TexCoord[0].st);" - " dx = (dx - 0.5);" - " dy = (dy - 0.5);" - " gl_FragColor = vec4(sqrt(dx*dx + dy*dy));" - "}"; - -/* horizontal convolution 3x3 */ -const gchar *hconv3_fragment_source = +const gchar *sep_sobel_length_fragment_source = + "#extension GL_ARB_texture_rectangle : enable\n" + "uniform sampler2DRect tex;" + "uniform bool invert;" + "void main () {" + " vec4 g = texture2DRect (tex, gl_TexCoord[0].st);" + /* restore black background with grey edges */ + " g -= vec4(0.5, 0.5, 0.0, 0.0);" + " float len = length (g);" + /* little trick to avoid IF operator */ + /* TODO: test if a standalone inverting pass is worth */ + " gl_FragColor = abs(int(invert) - vec4(vec3(len), 1.0));" + "}"; + +const gchar *desaturate_fragment_source = + "#extension GL_ARB_texture_rectangle : enable\n" + "uniform sampler2DRect tex;" + "void main () {" + " vec4 color = texture2DRect (tex, gl_TexCoord[0].st);" + " float luma = dot(color.rgb, vec3(0.2125, 0.7154, 0.0721));" + " gl_FragColor = vec4(vec3(luma), color.a);" + "}"; + +const gchar *sep_sobel_hconv3_fragment_source = "#extension GL_ARB_texture_rectangle : enable\n" "uniform sampler2DRect tex;" - "uniform float kernel[3];" - "uniform float offset;" "void main () {" " vec2 texturecoord[3];" - " float s = gl_TexCoord[0].s;" - " float t = gl_TexCoord[0].t;" - " texturecoord[0] = vec2(s-1.0, t);" - " texturecoord[1] = vec2(s, t);" - " texturecoord[2] = vec2(s+1.0, t);" + " texturecoord[1] = gl_TexCoord[0].st;" + " texturecoord[0] = texturecoord[1] - vec2(1.0, 0.0);" + " texturecoord[2] = texturecoord[1] + vec2(1.0, 0.0);" + " float grad_kern[3];" + " grad_kern[0] = 1.0;" + " grad_kern[1] = 0.0;" + " grad_kern[2] = -1.0;" + " float blur_kern[3];" + " blur_kern[0] = 0.25;" + " blur_kern[1] = 0.5;" + " blur_kern[2] = 0.25;" " int i;" " vec4 sum = vec4 (0.0);" " for (i = 0; i < 3; i++) { " " vec4 neighbor = texture2DRect(tex, texturecoord[i]); " - " sum += neighbor * kernel[i];" + " sum.r = neighbor.r * blur_kern[i] + sum.r;" + " sum.g = neighbor.g * grad_kern[i] + sum.g;" " }" - " gl_FragColor = sum + offset;" + " gl_FragColor = sum + vec4(0.0, 0.5, 0.0, 0.0);" "}"; -/* vertical convolution 3x3 */ -const gchar *vconv3_fragment_source = +const gchar *sep_sobel_vconv3_fragment_source = "#extension GL_ARB_texture_rectangle : enable\n" "uniform sampler2DRect tex;" - "uniform float kernel[3];" - "uniform float offset;" "void main () {" " vec2 texturecoord[3];" - " float s = gl_TexCoord[0].s;" - " float t = gl_TexCoord[0].t;" - " texturecoord[0] = vec2(s, t-1.0);" - " texturecoord[1] = vec2(s, t);" - " texturecoord[2] = vec2(s, t+1.0);" + " texturecoord[1] = gl_TexCoord[0].st;" + " texturecoord[0] = texturecoord[1] - vec2(0.0, 1.0);" + " texturecoord[2] = texturecoord[1] + vec2(0.0, 1.0);" + " float grad_kern[3];" + " grad_kern[0] = 1.0;" + " grad_kern[1] = 0.0;" + " grad_kern[2] = -1.0;" + " float blur_kern[3];" + " blur_kern[0] = 0.25;" + " blur_kern[1] = 0.5;" + " blur_kern[2] = 0.25;" " int i;" " vec4 sum = vec4 (0.0);" " for (i = 0; i < 3; i++) { " - " vec4 neighbor = texture2DRect(tex, texturecoord[i]);" - " sum += neighbor * kernel[i]; " + " vec4 neighbor = texture2DRect(tex, texturecoord[i]); " + " sum.r = neighbor.r * grad_kern[i] + sum.r;" + " sum.g = neighbor.g * blur_kern[i] + sum.g;" " }" - " gl_FragColor = sum + offset;" + " gl_FragColor = sum + vec4(0.5, 0.0, 0.0, 0.0);" "}"; /* horizontal convolution 9x9 */ diff --git a/gst/gl/effects/gstgleffectssources.h b/gst/gl/effects/gstgleffectssources.h index fc69b8b098..eebb87df5e 100644 --- a/gst/gl/effects/gstgleffectssources.h +++ b/gst/gl/effects/gstgleffectssources.h @@ -33,9 +33,10 @@ extern const gchar *bulge_fragment_source; extern const gchar *square_fragment_source; extern const gchar *luma_threshold_fragment_source; extern const gchar *sobel_fragment_source; -extern const gchar *sobel_gradient_length_fragment_source; -extern const gchar *hconv3_fragment_source; -extern const gchar *vconv3_fragment_source; +extern const gchar *sep_sobel_length_fragment_source; +extern const gchar *desaturate_fragment_source; +extern const gchar *sep_sobel_hconv3_fragment_source; +extern const gchar *sep_sobel_vconv3_fragment_source; extern const gchar *hconv9_fragment_source; extern const gchar *vconv9_fragment_source; extern const gchar *sum_fragment_source; diff --git a/gst/gl/gstglfiltersobel.c b/gst/gl/gstglfiltersobel.c index d2c2b93f31..535d971b60 100644 --- a/gst/gl/gstglfiltersobel.c +++ b/gst/gl/gstglfiltersobel.c @@ -65,24 +65,15 @@ static void gst_gl_filtersobel_draw_texture (GstGLFilterSobel * filtersobel, static void gst_gl_filtersobel_init_shader (GstGLFilter * filter); static gboolean gst_gl_filtersobel_filter (GstGLFilter * filter, GstGLBuffer * inbuf, GstGLBuffer * outbuf); -static void gst_gl_filtersobel_step_one (gint width, gint height, guint texture, - gpointer stuff); -static void gst_gl_filtersobel_step_two (gint width, gint height, guint texture, - gpointer stuff); -static void gst_gl_filtersobel_step_three (gint width, gint height, - guint texture, gpointer stuff); -static void gst_gl_filtersobel_step_four (gint width, gint height, - guint texture, gpointer stuff); -static void gst_gl_filtersobel_step_five (gint width, gint height, - guint texture, gpointer stuff); -static gfloat grad_kern[3] = { - 1.0, 0.0, -1.0, -}; - -static gfloat blur_kern[3] = { - 1.0 / 4.0, 2.0 / 4.0, 1.0 / 4.0, -}; +static void gst_gl_filtersobel_desaturate (gint width, gint height, + guint texture, gpointer stuff); +static void gst_gl_filtersobel_hconv (gint width, gint height, guint texture, + gpointer stuff); +static void gst_gl_filtersobel_vconv (gint width, gint height, guint texture, + gpointer stuff); +static void gst_gl_filtersobel_length (gint width, gint height, guint texture, + gpointer stuff); static void gst_gl_filtersobel_init_resources (GstGLFilter * filter) @@ -90,7 +81,7 @@ gst_gl_filtersobel_init_resources (GstGLFilter * filter) GstGLFilterSobel *filtersobel = GST_GL_FILTERSOBEL (filter); int i; - for (i = 0; i < 5; i++) { + for (i = 0; i < 2; i++) { glGenTextures (1, &filtersobel->midtexture[i]); glBindTexture (GL_TEXTURE_RECTANGLE_ARB, filtersobel->midtexture[i]); glTexImage2D (GL_TEXTURE_RECTANGLE_ARB, 0, GL_RGBA8, @@ -112,7 +103,7 @@ gst_gl_filtersobel_reset_resources (GstGLFilter * filter) GstGLFilterSobel *filtersobel = GST_GL_FILTERSOBEL (filter); int i; - for (i = 0; i < 5; i++) { + for (i = 0; i < 2; i++) { glDeleteTextures (1, &filtersobel->midtexture[i]); } } @@ -160,7 +151,7 @@ gst_gl_filtersobel_init (GstGLFilterSobel * filtersobel, filtersobel->hconv = NULL; filtersobel->vconv = NULL; filtersobel->invert = FALSE; - for (i = 0; i < 5; i++) { + for (i = 0; i < 2; i++) { filtersobel->midtexture[i] = 0; } } @@ -171,6 +162,7 @@ gst_gl_filter_filtersobel_reset (GstGLFilter * filter) GstGLFilterSobel *filtersobel = GST_GL_FILTERSOBEL (filter); //blocking call, wait the opengl thread has destroyed the shader + gst_gl_display_del_shader (filter->display, filtersobel->desat); gst_gl_display_del_shader (filter->display, filtersobel->hconv); gst_gl_display_del_shader (filter->display, filtersobel->vconv); gst_gl_display_del_shader (filter->display, filtersobel->len); @@ -214,12 +206,14 @@ gst_gl_filtersobel_init_shader (GstGLFilter * filter) GstGLFilterSobel *filtersobel = GST_GL_FILTERSOBEL (filter); //blocking call, wait the opengl thread has compiled the shader - gst_gl_display_gen_shader (filter->display, 0, hconv3_fragment_source, - &filtersobel->hconv); - gst_gl_display_gen_shader (filter->display, 0, vconv3_fragment_source, - &filtersobel->vconv); + gst_gl_display_gen_shader (filter->display, 0, desaturate_fragment_source, + &filtersobel->desat); gst_gl_display_gen_shader (filter->display, 0, - sobel_gradient_length_fragment_source, &filtersobel->len); + sep_sobel_hconv3_fragment_source, &filtersobel->hconv); + gst_gl_display_gen_shader (filter->display, 0, + sep_sobel_vconv3_fragment_source, &filtersobel->vconv); + gst_gl_display_gen_shader (filter->display, 0, + sep_sobel_length_fragment_source, &filtersobel->len); } static void @@ -252,21 +246,39 @@ gst_gl_filtersobel_filter (GstGLFilter * filter, GstGLBuffer * inbuf, GstGLFilterSobel *filtersobel = GST_GL_FILTERSOBEL (filter); gst_gl_filter_render_to_target (filter, inbuf->texture, - filtersobel->midtexture[0], gst_gl_filtersobel_step_one, filtersobel); + filtersobel->midtexture[0], gst_gl_filtersobel_desaturate, filtersobel); gst_gl_filter_render_to_target (filter, filtersobel->midtexture[0], - filtersobel->midtexture[1], gst_gl_filtersobel_step_two, filtersobel); - gst_gl_filter_render_to_target (filter, inbuf->texture, - filtersobel->midtexture[2], gst_gl_filtersobel_step_three, filtersobel); - gst_gl_filter_render_to_target (filter, filtersobel->midtexture[2], - filtersobel->midtexture[3], gst_gl_filtersobel_step_four, filtersobel); - gst_gl_filter_render_to_target (filter, filtersobel->midtexture[3], - outbuf->texture, gst_gl_filtersobel_step_five, filtersobel); - + filtersobel->midtexture[1], gst_gl_filtersobel_hconv, filtersobel); + gst_gl_filter_render_to_target (filter, filtersobel->midtexture[1], + filtersobel->midtexture[0], gst_gl_filtersobel_vconv, filtersobel); + gst_gl_filter_render_to_target (filter, filtersobel->midtexture[0], + outbuf->texture, gst_gl_filtersobel_length, filtersobel); return TRUE; } static void -gst_gl_filtersobel_step_one (gint width, gint height, guint texture, +gst_gl_filtersobel_desaturate (gint width, gint height, guint texture, + gpointer stuff) +{ + GstGLFilterSobel *filtersobel = GST_GL_FILTERSOBEL (stuff); + + glMatrixMode (GL_PROJECTION); + glLoadIdentity (); + + gst_gl_shader_use (filtersobel->desat); + + glActiveTexture (GL_TEXTURE1); + glEnable (GL_TEXTURE_RECTANGLE_ARB); + glBindTexture (GL_TEXTURE_RECTANGLE_ARB, texture); + glDisable (GL_TEXTURE_RECTANGLE_ARB); + + gst_gl_shader_set_uniform_1i (filtersobel->desat, "tex", 1); + + gst_gl_filtersobel_draw_texture (filtersobel, texture); +} + +static void +gst_gl_filtersobel_hconv (gint width, gint height, guint texture, gpointer stuff) { GstGLFilterSobel *filtersobel = GST_GL_FILTERSOBEL (stuff); @@ -282,14 +294,12 @@ gst_gl_filtersobel_step_one (gint width, gint height, guint texture, glDisable (GL_TEXTURE_RECTANGLE_ARB); gst_gl_shader_set_uniform_1i (filtersobel->hconv, "tex", 1); - gst_gl_shader_set_uniform_1fv (filtersobel->hconv, "kernel", 3, blur_kern); - gst_gl_shader_set_uniform_1f (filtersobel->hconv, "offset", 0.0); gst_gl_filtersobel_draw_texture (filtersobel, texture); } static void -gst_gl_filtersobel_step_two (gint width, gint height, guint texture, +gst_gl_filtersobel_vconv (gint width, gint height, guint texture, gpointer stuff) { GstGLFilterSobel *filtersobel = GST_GL_FILTERSOBEL (stuff); @@ -305,60 +315,12 @@ gst_gl_filtersobel_step_two (gint width, gint height, guint texture, glDisable (GL_TEXTURE_RECTANGLE_ARB); gst_gl_shader_set_uniform_1i (filtersobel->vconv, "tex", 1); - gst_gl_shader_set_uniform_1fv (filtersobel->vconv, "kernel", 3, grad_kern); - gst_gl_shader_set_uniform_1f (filtersobel->vconv, "offset", 0.5); gst_gl_filtersobel_draw_texture (filtersobel, texture); } static void -gst_gl_filtersobel_step_three (gint width, gint height, guint texture, - gpointer stuff) -{ - GstGLFilterSobel *filtersobel = GST_GL_FILTERSOBEL (stuff); - - glMatrixMode (GL_PROJECTION); - glLoadIdentity (); - - gst_gl_shader_use (filtersobel->vconv); - - glActiveTexture (GL_TEXTURE1); - glEnable (GL_TEXTURE_RECTANGLE_ARB); - glBindTexture (GL_TEXTURE_RECTANGLE_ARB, texture); - glDisable (GL_TEXTURE_RECTANGLE_ARB); - - gst_gl_shader_set_uniform_1i (filtersobel->vconv, "tex", 1); - gst_gl_shader_set_uniform_1fv (filtersobel->vconv, "kernel", 3, blur_kern); - gst_gl_shader_set_uniform_1f (filtersobel->vconv, "offset", 0.0); - - gst_gl_filtersobel_draw_texture (filtersobel, texture); -} - -static void -gst_gl_filtersobel_step_four (gint width, gint height, guint texture, - gpointer stuff) -{ - GstGLFilterSobel *filtersobel = GST_GL_FILTERSOBEL (stuff); - - glMatrixMode (GL_PROJECTION); - glLoadIdentity (); - - gst_gl_shader_use (filtersobel->hconv); - - glActiveTexture (GL_TEXTURE1); - glEnable (GL_TEXTURE_RECTANGLE_ARB); - glBindTexture (GL_TEXTURE_RECTANGLE_ARB, texture); - glDisable (GL_TEXTURE_RECTANGLE_ARB); - - gst_gl_shader_set_uniform_1i (filtersobel->hconv, "tex", 1); - gst_gl_shader_set_uniform_1fv (filtersobel->hconv, "kernel", 3, grad_kern); - gst_gl_shader_set_uniform_1f (filtersobel->vconv, "offset", 0.5); - - gst_gl_filtersobel_draw_texture (filtersobel, texture); -} - -static void -gst_gl_filtersobel_step_five (gint width, gint height, guint texture, +gst_gl_filtersobel_length (gint width, gint height, guint texture, gpointer stuff) { GstGLFilterSobel *filtersobel = GST_GL_FILTERSOBEL (stuff); @@ -370,18 +332,12 @@ gst_gl_filtersobel_step_five (gint width, gint height, guint texture, glActiveTexture (GL_TEXTURE1); glEnable (GL_TEXTURE_RECTANGLE_ARB); - glBindTexture (GL_TEXTURE_RECTANGLE_ARB, filtersobel->midtexture[1]); + glBindTexture (GL_TEXTURE_RECTANGLE_ARB, texture); glDisable (GL_TEXTURE_RECTANGLE_ARB); - gst_gl_shader_set_uniform_1i (filtersobel->len, "gx", 1); - - glActiveTexture (GL_TEXTURE2); - glEnable (GL_TEXTURE_RECTANGLE_ARB); - glBindTexture (GL_TEXTURE_RECTANGLE_ARB, filtersobel->midtexture[3]); - glDisable (GL_TEXTURE_RECTANGLE_ARB); - - gst_gl_shader_set_uniform_1i (filtersobel->len, "gy", 2); - + gst_gl_shader_set_uniform_1i (filtersobel->len, "tex", 1); + gst_gl_shader_set_uniform_1i (filtersobel->len, "invert", + filtersobel->invert); gst_gl_filtersobel_draw_texture (filtersobel, texture); } diff --git a/gst/gl/gstglfiltersobel.h b/gst/gl/gstglfiltersobel.h index 0d614f343d..faac6ac5c6 100644 --- a/gst/gl/gstglfiltersobel.h +++ b/gst/gl/gstglfiltersobel.h @@ -39,6 +39,7 @@ struct _GstGLFilterSobel GstGLShader *hconv; GstGLShader *vconv; GstGLShader *len; + GstGLShader *desat; GLuint midtexture[5];