Compare commits

...

5 Commits

View File

@ -65,346 +65,360 @@
* ]| will correct camera distortion once camera calibration is done. * ]| will correct camera distortion once camera calibration is done.
*/ */
#ifdef HAVE_CONFIG_H #ifdef HAVE_CONFIG_H
# include <config.h> # include <config.h>
#endif #endif
#include <vector> #include <vector>
#include "camerautils.hpp" #include "camerautils.hpp"
#include "cameraevent.hpp" #include "cameraevent.hpp"
#include "gstcameraundistort.h" #include "gstcameraundistort.h"
#include <opencv2/imgproc.hpp> #include <opencv2/imgproc.hpp>
#include <opencv2/calib3d.hpp> #include <opencv2/calib3d.hpp>
#include <gst/opencv/gstopencvutils.h> #include <gst/opencv/gstopencvutils.h>
GST_DEBUG_CATEGORY_STATIC (gst_camera_undistort_debug); #include <opencv2/core/ocl.hpp>
#define GST_CAT_DEFAULT gst_camera_undistort_debug
GST_DEBUG_CATEGORY_STATIC (gst_camera_undistort_debug);
#define DEFAULT_SHOW_UNDISTORTED TRUE #define GST_CAT_DEFAULT gst_camera_undistort_debug
#define DEFAULT_ALPHA 0.0
#define DEFAULT_CROP FALSE #define DEFAULT_SHOW_UNDISTORTED TRUE
#define DEFAULT_ALPHA 0.0
enum #define DEFAULT_CROP FALSE
{
PROP_0, enum
PROP_SHOW_UNDISTORTED, {
PROP_ALPHA, PROP_0,
PROP_CROP, PROP_SHOW_UNDISTORTED,
PROP_SETTINGS PROP_ALPHA,
}; PROP_CROP,
PROP_SETTINGS
G_DEFINE_TYPE_WITH_CODE (GstCameraUndistort, gst_camera_undistort, };
GST_TYPE_OPENCV_VIDEO_FILTER,
GST_DEBUG_CATEGORY_INIT (gst_camera_undistort_debug, "cameraundistort", 0, G_DEFINE_TYPE_WITH_CODE (GstCameraUndistort, gst_camera_undistort,
"Performs camera undistortion"); GST_TYPE_OPENCV_VIDEO_FILTER,
); GST_DEBUG_CATEGORY_INIT (gst_camera_undistort_debug, "cameraundistort", 0,
GST_ELEMENT_REGISTER_DEFINE (cameraundistort, "cameraundistort", GST_RANK_NONE, "Performs camera undistortion");
GST_TYPE_CAMERA_UNDISTORT); );
GST_ELEMENT_REGISTER_DEFINE (cameraundistort, "cameraundistort", GST_RANK_NONE,
static void gst_camera_undistort_dispose (GObject * object); GST_TYPE_CAMERA_UNDISTORT);
static void gst_camera_undistort_set_property (GObject * object, guint prop_id,
const GValue * value, GParamSpec * pspec); static void gst_camera_undistort_dispose (GObject * object);
static void gst_camera_undistort_get_property (GObject * object, guint prop_id, static void gst_camera_undistort_set_property (GObject * object, guint prop_id,
GValue * value, GParamSpec * pspec); const GValue * value, GParamSpec * pspec);
static void gst_camera_undistort_get_property (GObject * object, guint prop_id,
static gboolean gst_camera_undistort_set_info (GstOpencvVideoFilter * cvfilter, GValue * value, GParamSpec * pspec);
gint in_width, gint in_height, int in_cv_type,
gint out_width, gint out_height, int out_cv_type); static gboolean gst_camera_undistort_set_info (GstOpencvVideoFilter * cvfilter,
static GstFlowReturn gst_camera_undistort_transform_frame (GstOpencvVideoFilter gint in_width, gint in_height, int in_cv_type,
* cvfilter, GstBuffer * frame, cv::Mat img, GstBuffer * outframe, gint out_width, gint out_height, int out_cv_type);
cv::Mat outimg); static GstFlowReturn gst_camera_undistort_transform_frame (GstOpencvVideoFilter
* cvfilter, GstBuffer * frame, cv::Mat img, GstBuffer * outframe,
static gboolean gst_camera_undistort_sink_event (GstBaseTransform * trans, cv::Mat outimg);
GstEvent * event);
static gboolean gst_camera_undistort_src_event (GstBaseTransform * trans, static gboolean gst_camera_undistort_sink_event (GstBaseTransform * trans,
GstEvent * event); GstEvent * event);
static gboolean gst_camera_undistort_src_event (GstBaseTransform * trans,
static void camera_undistort_run (GstCameraUndistort * undist, cv::Mat img, GstEvent * event);
cv::Mat outimg);
static gboolean camera_undistort_init_undistort_rectify_map (GstCameraUndistort static void camera_undistort_run (GstCameraUndistort * undist, cv::Mat img,
* undist); cv::Mat outimg);
static gboolean camera_undistort_init_undistort_rectify_map (GstCameraUndistort
/* initialize the cameraundistort's class */ * undist);
static void
gst_camera_undistort_class_init (GstCameraUndistortClass * klass) /* initialize the cameraundistort's class */
{ static void
GObjectClass *gobject_class = G_OBJECT_CLASS (klass); gst_camera_undistort_class_init (GstCameraUndistortClass * klass)
GstElementClass *element_class = GST_ELEMENT_CLASS (klass); {
GstBaseTransformClass *trans_class = GST_BASE_TRANSFORM_CLASS (klass); GObjectClass *gobject_class = G_OBJECT_CLASS (klass);
GstOpencvVideoFilterClass *opencvfilter_class = GstElementClass *element_class = GST_ELEMENT_CLASS (klass);
GST_OPENCV_VIDEO_FILTER_CLASS (klass); GstBaseTransformClass *trans_class = GST_BASE_TRANSFORM_CLASS (klass);
GstOpencvVideoFilterClass *opencvfilter_class =
GstCaps *caps; GST_OPENCV_VIDEO_FILTER_CLASS (klass);
GstPadTemplate *templ;
GstCaps *caps;
gobject_class->dispose = gst_camera_undistort_dispose; GstPadTemplate *templ;
gobject_class->set_property = gst_camera_undistort_set_property;
gobject_class->get_property = gst_camera_undistort_get_property; gobject_class->dispose = gst_camera_undistort_dispose;
gobject_class->set_property = gst_camera_undistort_set_property;
trans_class->sink_event = GST_DEBUG_FUNCPTR (gst_camera_undistort_sink_event); gobject_class->get_property = gst_camera_undistort_get_property;
trans_class->src_event = GST_DEBUG_FUNCPTR (gst_camera_undistort_src_event);
trans_class->sink_event = GST_DEBUG_FUNCPTR (gst_camera_undistort_sink_event);
opencvfilter_class->cv_set_caps = gst_camera_undistort_set_info; trans_class->src_event = GST_DEBUG_FUNCPTR (gst_camera_undistort_src_event);
opencvfilter_class->cv_trans_func = gst_camera_undistort_transform_frame;
opencvfilter_class->cv_set_caps = gst_camera_undistort_set_info;
g_object_class_install_property (gobject_class, PROP_SHOW_UNDISTORTED, opencvfilter_class->cv_trans_func = gst_camera_undistort_transform_frame;
g_param_spec_boolean ("undistort", "Apply camera corrections",
"Apply camera corrections", g_object_class_install_property (gobject_class, PROP_SHOW_UNDISTORTED,
DEFAULT_SHOW_UNDISTORTED, g_param_spec_boolean ("undistort", "Apply camera corrections",
(GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS))); "Apply camera corrections",
DEFAULT_SHOW_UNDISTORTED,
g_object_class_install_property (gobject_class, PROP_ALPHA, (GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
g_param_spec_float ("alpha", "Pixels",
"Show all pixels (1), only valid ones (0) or something in between", g_object_class_install_property (gobject_class, PROP_ALPHA,
0.0, 1.0, DEFAULT_ALPHA, g_param_spec_float ("alpha", "Pixels",
(GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS))); "Show all pixels (1), only valid ones (0) or something in between",
0.0, 1.0, DEFAULT_ALPHA,
g_object_class_install_property (gobject_class, PROP_SETTINGS, (GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
g_param_spec_string ("settings", "Settings",
"Camera correction parameters (opaque string of serialized OpenCV objects)", g_object_class_install_property (gobject_class, PROP_SETTINGS,
NULL, (GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS))); g_param_spec_string ("settings", "Settings",
"Camera correction parameters (opaque string of serialized OpenCV objects)",
gst_element_class_set_static_metadata (element_class, NULL, (GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
"cameraundistort",
"Filter/Effect/Video", gst_element_class_set_static_metadata (element_class,
"Performs camera undistort", "Philippe Renon <philippe_renon@yahoo.fr>"); "cameraundistort",
"Filter/Effect/Video",
/* add sink and source pad templates */ "Performs camera undistort", "Philippe Renon <philippe_renon@yahoo.fr>");
caps = gst_opencv_caps_from_cv_image_type (CV_16UC1);
gst_caps_append (caps, gst_opencv_caps_from_cv_image_type (CV_8UC4)); /* add sink and source pad templates */
gst_caps_append (caps, gst_opencv_caps_from_cv_image_type (CV_8UC3)); caps = gst_opencv_caps_from_cv_image_type (CV_16UC1);
gst_caps_append (caps, gst_opencv_caps_from_cv_image_type (CV_8UC1)); gst_caps_append (caps, gst_opencv_caps_from_cv_image_type (CV_8UC4));
templ = gst_pad_template_new ("sink", GST_PAD_SINK, GST_PAD_ALWAYS, gst_caps_append (caps, gst_opencv_caps_from_cv_image_type (CV_8UC3));
gst_caps_ref (caps)); gst_caps_append (caps, gst_opencv_caps_from_cv_image_type (CV_8UC1));
gst_element_class_add_pad_template (element_class, templ); templ = gst_pad_template_new ("sink", GST_PAD_SINK, GST_PAD_ALWAYS,
templ = gst_pad_template_new ("src", GST_PAD_SRC, GST_PAD_ALWAYS, caps); gst_caps_ref (caps));
gst_element_class_add_pad_template (element_class, templ); gst_element_class_add_pad_template (element_class, templ);
} templ = gst_pad_template_new ("src", GST_PAD_SRC, GST_PAD_ALWAYS, caps);
gst_element_class_add_pad_template (element_class, templ);
/* initialize the new element }
* initialize instance structure
*/ /* initialize the new element
static void * initialize instance structure
gst_camera_undistort_init (GstCameraUndistort * undist) */
{ static void
undist->showUndistorted = DEFAULT_SHOW_UNDISTORTED; gst_camera_undistort_init (GstCameraUndistort * undist)
undist->alpha = DEFAULT_ALPHA; {
undist->crop = DEFAULT_CROP; undist->showUndistorted = DEFAULT_SHOW_UNDISTORTED;
undist->alpha = DEFAULT_ALPHA;
undist->doUndistort = FALSE; undist->crop = DEFAULT_CROP;
undist->settingsChanged = FALSE;
undist->doUndistort = FALSE;
undist->cameraMatrix = 0; undist->settingsChanged = FALSE;
undist->distCoeffs = 0;
undist->map1 = 0; undist->cameraMatrix = 0;
undist->map2 = 0; undist->distCoeffs = 0;
undist->map1 = 0;
undist->settings = NULL; undist->map2 = 0;
}
undist->settings = NULL;
static void }
gst_camera_undistort_dispose (GObject * object)
{ static void
GstCameraUndistort *undist = GST_CAMERA_UNDISTORT (object); gst_camera_undistort_dispose (GObject * object)
{
g_free (undist->settings); GstCameraUndistort *undist = GST_CAMERA_UNDISTORT (object);
undist->settings = NULL;
g_free (undist->settings);
G_OBJECT_CLASS (gst_camera_undistort_parent_class)->dispose (object); undist->settings = NULL;
}
G_OBJECT_CLASS (gst_camera_undistort_parent_class)->dispose (object);
}
static void
gst_camera_undistort_set_property (GObject * object, guint prop_id,
const GValue * value, GParamSpec * pspec) static void
{ gst_camera_undistort_set_property (GObject * object, guint prop_id,
GstCameraUndistort *undist = GST_CAMERA_UNDISTORT (object); const GValue * value, GParamSpec * pspec)
const char *str; {
GstCameraUndistort *undist = GST_CAMERA_UNDISTORT (object);
switch (prop_id) { const char *str;
case PROP_SHOW_UNDISTORTED:
undist->showUndistorted = g_value_get_boolean (value); switch (prop_id) {
undist->settingsChanged = TRUE; case PROP_SHOW_UNDISTORTED:
break; undist->showUndistorted = g_value_get_boolean (value);
case PROP_ALPHA: undist->settingsChanged = TRUE;
undist->alpha = g_value_get_float (value); break;
undist->settingsChanged = TRUE; case PROP_ALPHA:
break; undist->alpha = g_value_get_float (value);
case PROP_CROP: undist->settingsChanged = TRUE;
undist->crop = g_value_get_boolean (value); break;
break; case PROP_CROP:
case PROP_SETTINGS: undist->crop = g_value_get_boolean (value);
if (undist->settings) { break;
g_free (undist->settings); case PROP_SETTINGS:
undist->settings = NULL; if (undist->settings) {
} g_free (undist->settings);
str = g_value_get_string (value); undist->settings = NULL;
if (str) }
undist->settings = g_strdup (str); str = g_value_get_string (value);
undist->settingsChanged = TRUE; if (str)
break; undist->settings = g_strdup (str);
default: undist->settingsChanged = TRUE;
G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); break;
break; default:
} G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
} break;
}
static void }
gst_camera_undistort_get_property (GObject * object, guint prop_id,
GValue * value, GParamSpec * pspec) static void
{ gst_camera_undistort_get_property (GObject * object, guint prop_id,
GstCameraUndistort *undist = GST_CAMERA_UNDISTORT (object); GValue * value, GParamSpec * pspec)
{
switch (prop_id) { GstCameraUndistort *undist = GST_CAMERA_UNDISTORT (object);
case PROP_SHOW_UNDISTORTED:
g_value_set_boolean (value, undist->showUndistorted); switch (prop_id) {
break; case PROP_SHOW_UNDISTORTED:
case PROP_ALPHA: g_value_set_boolean (value, undist->showUndistorted);
g_value_set_float (value, undist->alpha); break;
break; case PROP_ALPHA:
case PROP_CROP: g_value_set_float (value, undist->alpha);
g_value_set_boolean (value, undist->crop); break;
break; case PROP_CROP:
case PROP_SETTINGS: g_value_set_boolean (value, undist->crop);
g_value_set_string (value, undist->settings); break;
break; case PROP_SETTINGS:
default: g_value_set_string (value, undist->settings);
G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); break;
break; default:
} G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
} break;
}
gboolean }
gst_camera_undistort_set_info (GstOpencvVideoFilter * cvfilter,
gint in_width, gint in_height, G_GNUC_UNUSED int in_cv_type, gboolean
G_GNUC_UNUSED gint out_width, G_GNUC_UNUSED gint out_height, gst_camera_undistort_set_info (GstOpencvVideoFilter * cvfilter,
G_GNUC_UNUSED int out_cv_type) gint in_width, gint in_height, G_GNUC_UNUSED int in_cv_type,
{ G_GNUC_UNUSED gint out_width, G_GNUC_UNUSED gint out_height,
GstCameraUndistort *undist = GST_CAMERA_UNDISTORT (cvfilter); G_GNUC_UNUSED int out_cv_type)
{
undist->imageSize = cv::Size (in_width, in_height); GstCameraUndistort *undist = GST_CAMERA_UNDISTORT (cvfilter);
return TRUE; undist->imageSize = cv::Size (in_width, in_height);
}
return TRUE;
/* }
* Performs the camera undistort
*/ /*
static GstFlowReturn * Performs the camera undistort
gst_camera_undistort_transform_frame (GstOpencvVideoFilter * cvfilter, */
G_GNUC_UNUSED GstBuffer * frame, cv::Mat img, static GstFlowReturn
G_GNUC_UNUSED GstBuffer * outframe, cv::Mat outimg) gst_camera_undistort_transform_frame (GstOpencvVideoFilter * cvfilter,
{ G_GNUC_UNUSED GstBuffer * frame, cv::Mat img,
GstCameraUndistort *undist = GST_CAMERA_UNDISTORT (cvfilter); G_GNUC_UNUSED GstBuffer * outframe, cv::Mat outimg)
{
camera_undistort_run (undist, img, outimg); GstCameraUndistort *undist = GST_CAMERA_UNDISTORT (cvfilter);
return GST_FLOW_OK; camera_undistort_run (undist, img, outimg);
}
return GST_FLOW_OK;
static void }
camera_undistort_run (GstCameraUndistort * undist, cv::Mat img, cv::Mat outimg)
{ static void
/* TODO is settingsChanged handling thread safe ? */ camera_undistort_run (GstCameraUndistort * undist, cv::Mat img, cv::Mat outimg)
if (undist->settingsChanged) { {
/* settings have changed, need to recompute undistort */ /* TODO is settingsChanged handling thread safe ? */
undist->settingsChanged = FALSE; if (undist->settingsChanged) {
undist->doUndistort = FALSE; /* settings have changed, need to recompute undistort */
if (undist->showUndistorted && undist->settings) { undist->settingsChanged = FALSE;
if (camera_deserialize_undistort_settings (undist->settings, undist->doUndistort = FALSE;
undist->cameraMatrix, undist->distCoeffs)) { if (undist->showUndistorted && undist->settings) {
undist->doUndistort = if (camera_deserialize_undistort_settings (undist->settings,
camera_undistort_init_undistort_rectify_map (undist); undist->cameraMatrix, undist->distCoeffs)) {
} undist->doUndistort =
} camera_undistort_init_undistort_rectify_map (undist);
} }
}
if (undist->showUndistorted && undist->doUndistort) { }
/* do the undistort */
cv::remap (img, outimg, undist->map1, undist->map2, cv::INTER_LINEAR); if (undist->showUndistorted && undist->doUndistort) {
if (undist->crop) { /* Declare input and output images as UMat to enable potential OpenCL acceleration. */
/* TODO do the cropping */ /* UMat allows OpenCV to automatically use GPU if available and configured. */
const cv::Scalar CROP_COLOR (0, 255, 0); cv::UMat cv_input_umat, cv_output_umat;
cv::rectangle (outimg, undist->validPixROI, CROP_COLOR);
} /* convert Mat to UMat */
} else { img.copyTo(cv_input_umat);
/* FIXME should use pass through to avoid this copy when not undistorting */
img.copyTo (outimg); /* do the undistort */
} cv::remap (cv_input_umat, cv_output_umat, undist->map1, undist->map2, cv::INTER_LINEAR);
}
/* UMat output back to Mat */
/* compute undistort */ cv_output_umat.copyTo(outimg);
static gboolean
camera_undistort_init_undistort_rectify_map (GstCameraUndistort * undist) if (undist->crop) {
{ /* TODO do the cropping */
cv::Size newImageSize; const cv::Scalar CROP_COLOR (0, 255, 0);
cv::Rect validPixROI; cv::rectangle (outimg, undist->validPixROI, CROP_COLOR);
cv::Mat newCameraMatrix = }
cv::getOptimalNewCameraMatrix (undist->cameraMatrix, undist->distCoeffs, } else {
undist->imageSize, undist->alpha, newImageSize, &validPixROI); /* FIXME should use pass through to avoid this copy when not undistorting */
undist->validPixROI = validPixROI; img.copyTo (outimg);
}
cv::initUndistortRectifyMap (undist->cameraMatrix, undist->distCoeffs, }
cv::Mat (), newCameraMatrix, undist->imageSize, CV_16SC2, undist->map1,
undist->map2); /* compute undistort */
static gboolean
return TRUE; camera_undistort_init_undistort_rectify_map (GstCameraUndistort * undist)
} {
cv::Size newImageSize;
static gboolean cv::Rect validPixROI;
camera_undistort_calibration_event (GstCameraUndistort * undist, cv::Mat newCameraMatrix =
GstEvent * event) cv::getOptimalNewCameraMatrix (undist->cameraMatrix, undist->distCoeffs,
{ undist->imageSize, undist->alpha, newImageSize, &validPixROI);
g_free (undist->settings); undist->validPixROI = validPixROI;
if (!gst_camera_event_parse_calibrated (event, &(undist->settings))) { cv::initUndistortRectifyMap (undist->cameraMatrix, undist->distCoeffs,
return FALSE; cv::Mat (), newCameraMatrix, undist->imageSize, CV_16SC2, undist->map1,
} undist->map2);
undist->settingsChanged = TRUE; return TRUE;
}
return TRUE;
} static gboolean
camera_undistort_calibration_event (GstCameraUndistort * undist,
static gboolean GstEvent * event)
gst_camera_undistort_sink_event (GstBaseTransform * trans, GstEvent * event) {
{ g_free (undist->settings);
GstCameraUndistort *undist = GST_CAMERA_UNDISTORT (trans);
if (!gst_camera_event_parse_calibrated (event, &(undist->settings))) {
const GstStructure *structure = gst_event_get_structure (event); return FALSE;
}
if (GST_EVENT_TYPE (event) == GST_EVENT_CUSTOM_BOTH && structure) {
if (strcmp (gst_structure_get_name (structure), undist->settingsChanged = TRUE;
GST_CAMERA_EVENT_CALIBRATED_NAME) == 0) {
return camera_undistort_calibration_event (undist, event); return TRUE;
} }
}
static gboolean
return gst_camera_undistort_sink_event (GstBaseTransform * trans, GstEvent * event)
GST_BASE_TRANSFORM_CLASS (gst_camera_undistort_parent_class)->sink_event {
(trans, event); GstCameraUndistort *undist = GST_CAMERA_UNDISTORT (trans);
}
const GstStructure *structure = gst_event_get_structure (event);
static gboolean
gst_camera_undistort_src_event (GstBaseTransform * trans, GstEvent * event) if (GST_EVENT_TYPE (event) == GST_EVENT_CUSTOM_BOTH && structure) {
{ if (strcmp (gst_structure_get_name (structure),
GstCameraUndistort *undist = GST_CAMERA_UNDISTORT (trans); GST_CAMERA_EVENT_CALIBRATED_NAME) == 0) {
return camera_undistort_calibration_event (undist, event);
const GstStructure *structure = gst_event_get_structure (event); }
}
if (GST_EVENT_TYPE (event) == GST_EVENT_CUSTOM_BOTH && structure) {
if (strcmp (gst_structure_get_name (structure), return
GST_CAMERA_EVENT_CALIBRATED_NAME) == 0) { GST_BASE_TRANSFORM_CLASS (gst_camera_undistort_parent_class)->sink_event
return camera_undistort_calibration_event (undist, event); (trans, event);
} }
}
static gboolean
return gst_camera_undistort_src_event (GstBaseTransform * trans, GstEvent * event)
GST_BASE_TRANSFORM_CLASS (gst_camera_undistort_parent_class)->src_event {
(trans, event); GstCameraUndistort *undist = GST_CAMERA_UNDISTORT (trans);
}
const GstStructure *structure = gst_event_get_structure (event);
if (GST_EVENT_TYPE (event) == GST_EVENT_CUSTOM_BOTH && structure) {
if (strcmp (gst_structure_get_name (structure),
GST_CAMERA_EVENT_CALIBRATED_NAME) == 0) {
return camera_undistort_calibration_event (undist, event);
}
}
return
GST_BASE_TRANSFORM_CLASS (gst_camera_undistort_parent_class)->src_event
(trans, event);
}