Compare commits

..

No commits in common. "cffd761d2e9c99acc591273eea898eb9ff3c4bf0" and "6377ebf568749de0bd07bafdd578cb85895a1ade" have entirely different histories.

View File

@ -65,360 +65,346 @@
* ]| will correct camera distortion once camera calibration is done. * ]| will correct camera distortion once camera calibration is done.
*/ */
#ifdef HAVE_CONFIG_H #ifdef HAVE_CONFIG_H
# include <config.h> # include <config.h>
#endif #endif
#include <vector> #include <vector>
#include "camerautils.hpp" #include "camerautils.hpp"
#include "cameraevent.hpp" #include "cameraevent.hpp"
#include "gstcameraundistort.h" #include "gstcameraundistort.h"
#include <opencv2/imgproc.hpp> #include <opencv2/imgproc.hpp>
#include <opencv2/calib3d.hpp> #include <opencv2/calib3d.hpp>
#include <gst/opencv/gstopencvutils.h> #include <gst/opencv/gstopencvutils.h>
#include <opencv2/core/ocl.hpp> GST_DEBUG_CATEGORY_STATIC (gst_camera_undistort_debug);
#define GST_CAT_DEFAULT gst_camera_undistort_debug
GST_DEBUG_CATEGORY_STATIC (gst_camera_undistort_debug);
#define GST_CAT_DEFAULT gst_camera_undistort_debug #define DEFAULT_SHOW_UNDISTORTED TRUE
#define DEFAULT_ALPHA 0.0
#define DEFAULT_SHOW_UNDISTORTED TRUE #define DEFAULT_CROP FALSE
#define DEFAULT_ALPHA 0.0
#define DEFAULT_CROP FALSE enum
{
enum PROP_0,
{ PROP_SHOW_UNDISTORTED,
PROP_0, PROP_ALPHA,
PROP_SHOW_UNDISTORTED, PROP_CROP,
PROP_ALPHA, PROP_SETTINGS
PROP_CROP, };
PROP_SETTINGS
}; G_DEFINE_TYPE_WITH_CODE (GstCameraUndistort, gst_camera_undistort,
GST_TYPE_OPENCV_VIDEO_FILTER,
G_DEFINE_TYPE_WITH_CODE (GstCameraUndistort, gst_camera_undistort, GST_DEBUG_CATEGORY_INIT (gst_camera_undistort_debug, "cameraundistort", 0,
GST_TYPE_OPENCV_VIDEO_FILTER, "Performs camera undistortion");
GST_DEBUG_CATEGORY_INIT (gst_camera_undistort_debug, "cameraundistort", 0, );
"Performs camera undistortion"); GST_ELEMENT_REGISTER_DEFINE (cameraundistort, "cameraundistort", GST_RANK_NONE,
); GST_TYPE_CAMERA_UNDISTORT);
GST_ELEMENT_REGISTER_DEFINE (cameraundistort, "cameraundistort", GST_RANK_NONE,
GST_TYPE_CAMERA_UNDISTORT); static void gst_camera_undistort_dispose (GObject * object);
static void gst_camera_undistort_set_property (GObject * object, guint prop_id,
static void gst_camera_undistort_dispose (GObject * object); const GValue * value, GParamSpec * pspec);
static void gst_camera_undistort_set_property (GObject * object, guint prop_id, static void gst_camera_undistort_get_property (GObject * object, guint prop_id,
const GValue * value, GParamSpec * pspec); GValue * value, GParamSpec * pspec);
static void gst_camera_undistort_get_property (GObject * object, guint prop_id,
GValue * value, GParamSpec * pspec); static gboolean gst_camera_undistort_set_info (GstOpencvVideoFilter * cvfilter,
gint in_width, gint in_height, int in_cv_type,
static gboolean gst_camera_undistort_set_info (GstOpencvVideoFilter * cvfilter, gint out_width, gint out_height, int out_cv_type);
gint in_width, gint in_height, int in_cv_type, static GstFlowReturn gst_camera_undistort_transform_frame (GstOpencvVideoFilter
gint out_width, gint out_height, int out_cv_type); * cvfilter, GstBuffer * frame, cv::Mat img, GstBuffer * outframe,
static GstFlowReturn gst_camera_undistort_transform_frame (GstOpencvVideoFilter cv::Mat outimg);
* cvfilter, GstBuffer * frame, cv::Mat img, GstBuffer * outframe,
cv::Mat outimg); static gboolean gst_camera_undistort_sink_event (GstBaseTransform * trans,
GstEvent * event);
static gboolean gst_camera_undistort_sink_event (GstBaseTransform * trans, static gboolean gst_camera_undistort_src_event (GstBaseTransform * trans,
GstEvent * event); GstEvent * event);
static gboolean gst_camera_undistort_src_event (GstBaseTransform * trans,
GstEvent * event); static void camera_undistort_run (GstCameraUndistort * undist, cv::Mat img,
cv::Mat outimg);
static void camera_undistort_run (GstCameraUndistort * undist, cv::Mat img, static gboolean camera_undistort_init_undistort_rectify_map (GstCameraUndistort
cv::Mat outimg); * undist);
static gboolean camera_undistort_init_undistort_rectify_map (GstCameraUndistort
* undist); /* initialize the cameraundistort's class */
static void
/* initialize the cameraundistort's class */ gst_camera_undistort_class_init (GstCameraUndistortClass * klass)
static void {
gst_camera_undistort_class_init (GstCameraUndistortClass * klass) GObjectClass *gobject_class = G_OBJECT_CLASS (klass);
{ GstElementClass *element_class = GST_ELEMENT_CLASS (klass);
GObjectClass *gobject_class = G_OBJECT_CLASS (klass); GstBaseTransformClass *trans_class = GST_BASE_TRANSFORM_CLASS (klass);
GstElementClass *element_class = GST_ELEMENT_CLASS (klass); GstOpencvVideoFilterClass *opencvfilter_class =
GstBaseTransformClass *trans_class = GST_BASE_TRANSFORM_CLASS (klass); GST_OPENCV_VIDEO_FILTER_CLASS (klass);
GstOpencvVideoFilterClass *opencvfilter_class =
GST_OPENCV_VIDEO_FILTER_CLASS (klass); GstCaps *caps;
GstPadTemplate *templ;
GstCaps *caps;
GstPadTemplate *templ; gobject_class->dispose = gst_camera_undistort_dispose;
gobject_class->set_property = gst_camera_undistort_set_property;
gobject_class->dispose = gst_camera_undistort_dispose; gobject_class->get_property = gst_camera_undistort_get_property;
gobject_class->set_property = gst_camera_undistort_set_property;
gobject_class->get_property = gst_camera_undistort_get_property; trans_class->sink_event = GST_DEBUG_FUNCPTR (gst_camera_undistort_sink_event);
trans_class->src_event = GST_DEBUG_FUNCPTR (gst_camera_undistort_src_event);
trans_class->sink_event = GST_DEBUG_FUNCPTR (gst_camera_undistort_sink_event);
trans_class->src_event = GST_DEBUG_FUNCPTR (gst_camera_undistort_src_event); opencvfilter_class->cv_set_caps = gst_camera_undistort_set_info;
opencvfilter_class->cv_trans_func = gst_camera_undistort_transform_frame;
opencvfilter_class->cv_set_caps = gst_camera_undistort_set_info;
opencvfilter_class->cv_trans_func = gst_camera_undistort_transform_frame; g_object_class_install_property (gobject_class, PROP_SHOW_UNDISTORTED,
g_param_spec_boolean ("undistort", "Apply camera corrections",
g_object_class_install_property (gobject_class, PROP_SHOW_UNDISTORTED, "Apply camera corrections",
g_param_spec_boolean ("undistort", "Apply camera corrections", DEFAULT_SHOW_UNDISTORTED,
"Apply camera corrections", (GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
DEFAULT_SHOW_UNDISTORTED,
(GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS))); g_object_class_install_property (gobject_class, PROP_ALPHA,
g_param_spec_float ("alpha", "Pixels",
g_object_class_install_property (gobject_class, PROP_ALPHA, "Show all pixels (1), only valid ones (0) or something in between",
g_param_spec_float ("alpha", "Pixels", 0.0, 1.0, DEFAULT_ALPHA,
"Show all pixels (1), only valid ones (0) or something in between", (GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
0.0, 1.0, DEFAULT_ALPHA,
(GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS))); g_object_class_install_property (gobject_class, PROP_SETTINGS,
g_param_spec_string ("settings", "Settings",
g_object_class_install_property (gobject_class, PROP_SETTINGS, "Camera correction parameters (opaque string of serialized OpenCV objects)",
g_param_spec_string ("settings", "Settings", NULL, (GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
"Camera correction parameters (opaque string of serialized OpenCV objects)",
NULL, (GParamFlags) (G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS))); gst_element_class_set_static_metadata (element_class,
"cameraundistort",
gst_element_class_set_static_metadata (element_class, "Filter/Effect/Video",
"cameraundistort", "Performs camera undistort", "Philippe Renon <philippe_renon@yahoo.fr>");
"Filter/Effect/Video",
"Performs camera undistort", "Philippe Renon <philippe_renon@yahoo.fr>"); /* add sink and source pad templates */
caps = gst_opencv_caps_from_cv_image_type (CV_16UC1);
/* add sink and source pad templates */ gst_caps_append (caps, gst_opencv_caps_from_cv_image_type (CV_8UC4));
caps = gst_opencv_caps_from_cv_image_type (CV_16UC1); gst_caps_append (caps, gst_opencv_caps_from_cv_image_type (CV_8UC3));
gst_caps_append (caps, gst_opencv_caps_from_cv_image_type (CV_8UC4)); gst_caps_append (caps, gst_opencv_caps_from_cv_image_type (CV_8UC1));
gst_caps_append (caps, gst_opencv_caps_from_cv_image_type (CV_8UC3)); templ = gst_pad_template_new ("sink", GST_PAD_SINK, GST_PAD_ALWAYS,
gst_caps_append (caps, gst_opencv_caps_from_cv_image_type (CV_8UC1)); gst_caps_ref (caps));
templ = gst_pad_template_new ("sink", GST_PAD_SINK, GST_PAD_ALWAYS, gst_element_class_add_pad_template (element_class, templ);
gst_caps_ref (caps)); templ = gst_pad_template_new ("src", GST_PAD_SRC, GST_PAD_ALWAYS, caps);
gst_element_class_add_pad_template (element_class, templ); gst_element_class_add_pad_template (element_class, templ);
templ = gst_pad_template_new ("src", GST_PAD_SRC, GST_PAD_ALWAYS, caps); }
gst_element_class_add_pad_template (element_class, templ);
} /* initialize the new element
* initialize instance structure
/* initialize the new element */
* initialize instance structure static void
*/ gst_camera_undistort_init (GstCameraUndistort * undist)
static void {
gst_camera_undistort_init (GstCameraUndistort * undist) undist->showUndistorted = DEFAULT_SHOW_UNDISTORTED;
{ undist->alpha = DEFAULT_ALPHA;
undist->showUndistorted = DEFAULT_SHOW_UNDISTORTED; undist->crop = DEFAULT_CROP;
undist->alpha = DEFAULT_ALPHA;
undist->crop = DEFAULT_CROP; undist->doUndistort = FALSE;
undist->settingsChanged = FALSE;
undist->doUndistort = FALSE;
undist->settingsChanged = FALSE; undist->cameraMatrix = 0;
undist->distCoeffs = 0;
undist->cameraMatrix = 0; undist->map1 = 0;
undist->distCoeffs = 0; undist->map2 = 0;
undist->map1 = 0;
undist->map2 = 0; undist->settings = NULL;
}
undist->settings = NULL;
} static void
gst_camera_undistort_dispose (GObject * object)
static void {
gst_camera_undistort_dispose (GObject * object) GstCameraUndistort *undist = GST_CAMERA_UNDISTORT (object);
{
GstCameraUndistort *undist = GST_CAMERA_UNDISTORT (object); g_free (undist->settings);
undist->settings = NULL;
g_free (undist->settings);
undist->settings = NULL; G_OBJECT_CLASS (gst_camera_undistort_parent_class)->dispose (object);
}
G_OBJECT_CLASS (gst_camera_undistort_parent_class)->dispose (object);
}
static void
gst_camera_undistort_set_property (GObject * object, guint prop_id,
static void const GValue * value, GParamSpec * pspec)
gst_camera_undistort_set_property (GObject * object, guint prop_id, {
const GValue * value, GParamSpec * pspec) GstCameraUndistort *undist = GST_CAMERA_UNDISTORT (object);
{ const char *str;
GstCameraUndistort *undist = GST_CAMERA_UNDISTORT (object);
const char *str; switch (prop_id) {
case PROP_SHOW_UNDISTORTED:
switch (prop_id) { undist->showUndistorted = g_value_get_boolean (value);
case PROP_SHOW_UNDISTORTED: undist->settingsChanged = TRUE;
undist->showUndistorted = g_value_get_boolean (value); break;
undist->settingsChanged = TRUE; case PROP_ALPHA:
break; undist->alpha = g_value_get_float (value);
case PROP_ALPHA: undist->settingsChanged = TRUE;
undist->alpha = g_value_get_float (value); break;
undist->settingsChanged = TRUE; case PROP_CROP:
break; undist->crop = g_value_get_boolean (value);
case PROP_CROP: break;
undist->crop = g_value_get_boolean (value); case PROP_SETTINGS:
break; if (undist->settings) {
case PROP_SETTINGS: g_free (undist->settings);
if (undist->settings) { undist->settings = NULL;
g_free (undist->settings); }
undist->settings = NULL; str = g_value_get_string (value);
} if (str)
str = g_value_get_string (value); undist->settings = g_strdup (str);
if (str) undist->settingsChanged = TRUE;
undist->settings = g_strdup (str); break;
undist->settingsChanged = TRUE; default:
break; G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
default: break;
G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); }
break; }
}
} static void
gst_camera_undistort_get_property (GObject * object, guint prop_id,
static void GValue * value, GParamSpec * pspec)
gst_camera_undistort_get_property (GObject * object, guint prop_id, {
GValue * value, GParamSpec * pspec) GstCameraUndistort *undist = GST_CAMERA_UNDISTORT (object);
{
GstCameraUndistort *undist = GST_CAMERA_UNDISTORT (object); switch (prop_id) {
case PROP_SHOW_UNDISTORTED:
switch (prop_id) { g_value_set_boolean (value, undist->showUndistorted);
case PROP_SHOW_UNDISTORTED: break;
g_value_set_boolean (value, undist->showUndistorted); case PROP_ALPHA:
break; g_value_set_float (value, undist->alpha);
case PROP_ALPHA: break;
g_value_set_float (value, undist->alpha); case PROP_CROP:
break; g_value_set_boolean (value, undist->crop);
case PROP_CROP: break;
g_value_set_boolean (value, undist->crop); case PROP_SETTINGS:
break; g_value_set_string (value, undist->settings);
case PROP_SETTINGS: break;
g_value_set_string (value, undist->settings); default:
break; G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
default: break;
G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); }
break; }
}
} gboolean
gst_camera_undistort_set_info (GstOpencvVideoFilter * cvfilter,
gboolean gint in_width, gint in_height, G_GNUC_UNUSED int in_cv_type,
gst_camera_undistort_set_info (GstOpencvVideoFilter * cvfilter, G_GNUC_UNUSED gint out_width, G_GNUC_UNUSED gint out_height,
gint in_width, gint in_height, G_GNUC_UNUSED int in_cv_type, G_GNUC_UNUSED int out_cv_type)
G_GNUC_UNUSED gint out_width, G_GNUC_UNUSED gint out_height, {
G_GNUC_UNUSED int out_cv_type) GstCameraUndistort *undist = GST_CAMERA_UNDISTORT (cvfilter);
{
GstCameraUndistort *undist = GST_CAMERA_UNDISTORT (cvfilter); undist->imageSize = cv::Size (in_width, in_height);
undist->imageSize = cv::Size (in_width, in_height); return TRUE;
}
return TRUE;
} /*
* Performs the camera undistort
/* */
* Performs the camera undistort static GstFlowReturn
*/ gst_camera_undistort_transform_frame (GstOpencvVideoFilter * cvfilter,
static GstFlowReturn G_GNUC_UNUSED GstBuffer * frame, cv::Mat img,
gst_camera_undistort_transform_frame (GstOpencvVideoFilter * cvfilter, G_GNUC_UNUSED GstBuffer * outframe, cv::Mat outimg)
G_GNUC_UNUSED GstBuffer * frame, cv::Mat img, {
G_GNUC_UNUSED GstBuffer * outframe, cv::Mat outimg) GstCameraUndistort *undist = GST_CAMERA_UNDISTORT (cvfilter);
{
GstCameraUndistort *undist = GST_CAMERA_UNDISTORT (cvfilter); camera_undistort_run (undist, img, outimg);
camera_undistort_run (undist, img, outimg); return GST_FLOW_OK;
}
return GST_FLOW_OK;
} static void
camera_undistort_run (GstCameraUndistort * undist, cv::Mat img, cv::Mat outimg)
static void {
camera_undistort_run (GstCameraUndistort * undist, cv::Mat img, cv::Mat outimg) /* TODO is settingsChanged handling thread safe ? */
{ if (undist->settingsChanged) {
/* TODO is settingsChanged handling thread safe ? */ /* settings have changed, need to recompute undistort */
if (undist->settingsChanged) { undist->settingsChanged = FALSE;
/* settings have changed, need to recompute undistort */ undist->doUndistort = FALSE;
undist->settingsChanged = FALSE; if (undist->showUndistorted && undist->settings) {
undist->doUndistort = FALSE; if (camera_deserialize_undistort_settings (undist->settings,
if (undist->showUndistorted && undist->settings) { undist->cameraMatrix, undist->distCoeffs)) {
if (camera_deserialize_undistort_settings (undist->settings, undist->doUndistort =
undist->cameraMatrix, undist->distCoeffs)) { camera_undistort_init_undistort_rectify_map (undist);
undist->doUndistort = }
camera_undistort_init_undistort_rectify_map (undist); }
} }
}
} if (undist->showUndistorted && undist->doUndistort) {
/* do the undistort */
if (undist->showUndistorted && undist->doUndistort) { cv::remap (img, outimg, undist->map1, undist->map2, cv::INTER_LINEAR);
/* Declare input and output images as UMat to enable potential OpenCL acceleration. */ if (undist->crop) {
/* UMat allows OpenCV to automatically use GPU if available and configured. */ /* TODO do the cropping */
cv::UMat cv_input_umat, cv_output_umat; const cv::Scalar CROP_COLOR (0, 255, 0);
cv::rectangle (outimg, undist->validPixROI, CROP_COLOR);
/* convert Mat to UMat */ }
img.copyTo(cv_input_umat); } else {
/* FIXME should use pass through to avoid this copy when not undistorting */
/* do the undistort */ img.copyTo (outimg);
cv::remap (cv_input_umat, cv_output_umat, undist->map1, undist->map2, cv::INTER_LINEAR); }
}
/* UMat output back to Mat */
cv_output_umat.copyTo(outimg); /* compute undistort */
static gboolean
if (undist->crop) { camera_undistort_init_undistort_rectify_map (GstCameraUndistort * undist)
/* TODO do the cropping */ {
const cv::Scalar CROP_COLOR (0, 255, 0); cv::Size newImageSize;
cv::rectangle (outimg, undist->validPixROI, CROP_COLOR); cv::Rect validPixROI;
} cv::Mat newCameraMatrix =
} else { cv::getOptimalNewCameraMatrix (undist->cameraMatrix, undist->distCoeffs,
/* FIXME should use pass through to avoid this copy when not undistorting */ undist->imageSize, undist->alpha, newImageSize, &validPixROI);
img.copyTo (outimg); undist->validPixROI = validPixROI;
}
} cv::initUndistortRectifyMap (undist->cameraMatrix, undist->distCoeffs,
cv::Mat (), newCameraMatrix, undist->imageSize, CV_16SC2, undist->map1,
/* compute undistort */ undist->map2);
static gboolean
camera_undistort_init_undistort_rectify_map (GstCameraUndistort * undist) return TRUE;
{ }
cv::Size newImageSize;
cv::Rect validPixROI; static gboolean
cv::Mat newCameraMatrix = camera_undistort_calibration_event (GstCameraUndistort * undist,
cv::getOptimalNewCameraMatrix (undist->cameraMatrix, undist->distCoeffs, GstEvent * event)
undist->imageSize, undist->alpha, newImageSize, &validPixROI); {
undist->validPixROI = validPixROI; g_free (undist->settings);
cv::initUndistortRectifyMap (undist->cameraMatrix, undist->distCoeffs, if (!gst_camera_event_parse_calibrated (event, &(undist->settings))) {
cv::Mat (), newCameraMatrix, undist->imageSize, CV_16SC2, undist->map1, return FALSE;
undist->map2); }
return TRUE; undist->settingsChanged = TRUE;
}
return TRUE;
static gboolean }
camera_undistort_calibration_event (GstCameraUndistort * undist,
GstEvent * event) static gboolean
{ gst_camera_undistort_sink_event (GstBaseTransform * trans, GstEvent * event)
g_free (undist->settings); {
GstCameraUndistort *undist = GST_CAMERA_UNDISTORT (trans);
if (!gst_camera_event_parse_calibrated (event, &(undist->settings))) {
return FALSE; const GstStructure *structure = gst_event_get_structure (event);
}
if (GST_EVENT_TYPE (event) == GST_EVENT_CUSTOM_BOTH && structure) {
undist->settingsChanged = TRUE; if (strcmp (gst_structure_get_name (structure),
GST_CAMERA_EVENT_CALIBRATED_NAME) == 0) {
return TRUE; return camera_undistort_calibration_event (undist, event);
} }
}
static gboolean
gst_camera_undistort_sink_event (GstBaseTransform * trans, GstEvent * event) return
{ GST_BASE_TRANSFORM_CLASS (gst_camera_undistort_parent_class)->sink_event
GstCameraUndistort *undist = GST_CAMERA_UNDISTORT (trans); (trans, event);
}
const GstStructure *structure = gst_event_get_structure (event);
static gboolean
if (GST_EVENT_TYPE (event) == GST_EVENT_CUSTOM_BOTH && structure) { gst_camera_undistort_src_event (GstBaseTransform * trans, GstEvent * event)
if (strcmp (gst_structure_get_name (structure), {
GST_CAMERA_EVENT_CALIBRATED_NAME) == 0) { GstCameraUndistort *undist = GST_CAMERA_UNDISTORT (trans);
return camera_undistort_calibration_event (undist, event);
} const GstStructure *structure = gst_event_get_structure (event);
}
if (GST_EVENT_TYPE (event) == GST_EVENT_CUSTOM_BOTH && structure) {
return if (strcmp (gst_structure_get_name (structure),
GST_BASE_TRANSFORM_CLASS (gst_camera_undistort_parent_class)->sink_event GST_CAMERA_EVENT_CALIBRATED_NAME) == 0) {
(trans, event); return camera_undistort_calibration_event (undist, event);
} }
}
static gboolean
gst_camera_undistort_src_event (GstBaseTransform * trans, GstEvent * event) return
{ GST_BASE_TRANSFORM_CLASS (gst_camera_undistort_parent_class)->src_event
GstCameraUndistort *undist = GST_CAMERA_UNDISTORT (trans); (trans, event);
}
const GstStructure *structure = gst_event_get_structure (event);
if (GST_EVENT_TYPE (event) == GST_EVENT_CUSTOM_BOTH && structure) {
if (strcmp (gst_structure_get_name (structure),
GST_CAMERA_EVENT_CALIBRATED_NAME) == 0) {
return camera_undistort_calibration_event (undist, event);
}
}
return
GST_BASE_TRANSFORM_CLASS (gst_camera_undistort_parent_class)->src_event
(trans, event);
}