From f1f8e465f27d41f55524fa5c6cfc90d798a310bd Mon Sep 17 00:00:00 2001 From: Nirbheek Chauhan Date: Sat, 5 Sep 2015 01:50:41 +0530 Subject: [PATCH] compositor: Ensure all arguments to CLAMP are signed int If any of the arguments to CLAMP are unsigned integers, the comparison causes an automatic conversion of the signed int to unsigned, which causes -1 to become UINT_MAX and get clamped to the high value of the CLAMP instead of 0. See 716 at http://c0x.coding-guidelines.com/6.3.1.8.html Also add a test for this. https://bugzilla.gnome.org/show_bug.cgi?id=754576 --- gst/compositor/compositor.c | 4 ++-- tests/check/elements/compositor.c | 11 +++++++++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/gst/compositor/compositor.c b/gst/compositor/compositor.c index 64d05ac180..d015d85e10 100644 --- a/gst/compositor/compositor.c +++ b/gst/compositor/compositor.c @@ -351,8 +351,8 @@ is_rectangle_contained (GstVideoRectangle rect1, GstVideoRectangle rect2) } static GstVideoRectangle -clamp_rectangle (gint x, gint y, gint w, gint h, guint outer_width, - guint outer_height) +clamp_rectangle (gint x, gint y, gint w, gint h, gint outer_width, + gint outer_height) { gint x2 = x + w; gint y2 = y + h; diff --git a/tests/check/elements/compositor.c b/tests/check/elements/compositor.c index dc1a58f031..ccd68a1970 100644 --- a/tests/check/elements/compositor.c +++ b/tests/check/elements/compositor.c @@ -1326,6 +1326,17 @@ GST_START_TEST (test_obscured_skipped) out_width = out_height = 0; buffer_mapped = FALSE; + xpos1 = -1; + xpos0 = ypos0 = width0 = height0 = width1 = height1 = 10; + out_width = out_height = 20; + GST_INFO ("testing bug 754576"); + _test_obscured (caps_str, xpos0, ypos0, width0, height0, alpha0, xpos1, ypos1, + width1, height1, alpha1, out_width, out_height); + fail_unless (buffer_mapped == TRUE); + xpos0 = xpos1 = ypos1 = width0 = height0 = width1 = height1 = 0; + out_width = out_height = 0; + buffer_mapped = FALSE; + xpos0 = ypos0 = 10000; out_width = 320; out_height = 240;