FaSTTUBe/GST-Tensordecoder-ov_ep
37
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

remove refereces of old decoders

Browse Source
This commit is contained in:
Elias Rosendahl 2025-09-01 18:02:52 +02:00
parent 52dfe8c17a
commit 8acdb1f8d3
2 changed files with 86 additions and 111 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

192
subprojects/gst-plugins-bad/gst/tensordecoders/gstssdobjectdetector.c
View File

@ -53,10 +53,10 @@
#include <gst/analytics/analytics.h>
/* Object detection tensor id strings */
#define GST_MODEL_OBJECT_DETECTOR_BOXES "ssd-mobilenet-v1-variant-1-out-boxes"
#define GST_MODEL_OBJECT_DETECTOR_SCORES "ssd-mobilenet-v1-variant-1-out-scores"
#define GST_MODEL_OBJECT_DETECTOR_NUM_DETECTIONS "generic-variant-1-out-count"
#define GST_MODEL_OBJECT_DETECTOR_CLASSES "ssd-mobilenet-v1-variant-1-out-classes"
#define GST_MODEL_OBJECT_DETECTOR_BOXES "Gst.Model.ObjectDetector.Boxes"
#define GST_MODEL_OBJECT_DETECTOR_SCORES "Gst.Model.ObjectDetector.Scores"
#define GST_MODEL_OBJECT_DETECTOR_NUM_DETECTIONS "Gst.Model.ObjectDetector.NumDetections"
#define GST_MODEL_OBJECT_DETECTOR_CLASSES "Gst.Model.ObjectDetector.Classes"
GST_DEBUG_CATEGORY_STATIC (ssd_object_detector_debug);
#define GST_CAT_DEFAULT ssd_object_detector_debug
@ -161,7 +161,7 @@ gst_ssd_object_detector_class_init (GstSsdObjectDetectorClass * klass)
(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)));
gst_element_class_set_static_metadata (element_class, "objectdetector",
"Tensordecoder/Video",
"TensorDecoder/Video",
"Apply tensor output from inference to detect objects in video frames",
"Aaron Boxer <aaron.boxer@collabora.com>, Marcus Edel <marcus.edel@collabora.com>");
gst_element_class_add_pad_template (element_class,
@ -306,80 +306,48 @@ gst_ssd_object_detector_get_property (GObject * object, guint prop_id,
}
}
static gboolean
gst_ssd_object_detector_get_tensors (GstSsdObjectDetector * object_detector,
GstBuffer * buf, const GstTensor ** classes_tensor,
const GstTensor ** numdetect_tensor, const GstTensor ** scores_tensor,
const GstTensor ** boxes_tensor)
static GstTensorMeta *
gst_ssd_object_detector_get_tensor_meta (GstSsdObjectDetector * object_detector,
GstBuffer * buf)
{
GstMeta *meta = NULL;
gpointer iter_state = NULL;
static const gsize BOXES_DIMS[] = { 1, G_MAXSIZE, 4 };
static const gsize NUM_DETECT_DIMS[] = { 1 };
static const gsize SCORES_CLASSES_DIMS[] = { 1, G_MAXSIZE };
if (!gst_buffer_get_meta (buf, GST_TENSOR_META_API_TYPE)) {
GST_DEBUG_OBJECT (object_detector,
"missing tensor meta from buffer %" GST_PTR_FORMAT, buf);
return FALSE;
return NULL;
}
// find object detector meta
while ((meta = gst_buffer_iterate_meta_filtered (buf, &iter_state,
GST_TENSOR_META_API_TYPE))) {
GstTensorMeta *tmeta = (GstTensorMeta *) meta;
*boxes_tensor = gst_tensor_meta_get_typed_tensor (tmeta,
g_quark_from_static_string (GST_MODEL_OBJECT_DETECTOR_BOXES),
GST_TENSOR_DATA_TYPE_FLOAT32, GST_TENSOR_DIM_ORDER_ROW_MAJOR, 3,
BOXES_DIMS);
if (*boxes_tensor == NULL)
*boxes_tensor = gst_tensor_meta_get_typed_tensor (tmeta,
g_quark_from_static_string (GST_MODEL_OBJECT_DETECTOR_BOXES),
GST_TENSOR_DATA_TYPE_UINT32, GST_TENSOR_DIM_ORDER_ROW_MAJOR, 3,
BOXES_DIMS);
if (*boxes_tensor == NULL)
GstTensorMeta *tensor_meta = (GstTensorMeta *) meta;
/* SSD model must have either 3 or 4 output tensor nodes: 4 if there is a label node,
* and only 3 if there is no label */
if (tensor_meta->num_tensors != 3 && tensor_meta->num_tensors != 4)
continue;
*scores_tensor = gst_tensor_meta_get_typed_tensor (tmeta,
g_quark_from_static_string (GST_MODEL_OBJECT_DETECTOR_SCORES),
GST_TENSOR_DATA_TYPE_FLOAT32, GST_TENSOR_DIM_ORDER_ROW_MAJOR, 2,
SCORES_CLASSES_DIMS);
if (*scores_tensor == NULL)
*scores_tensor = gst_tensor_meta_get_typed_tensor (tmeta,
g_quark_from_static_string (GST_MODEL_OBJECT_DETECTOR_SCORES),
GST_TENSOR_DATA_TYPE_UINT32, GST_TENSOR_DIM_ORDER_ROW_MAJOR, 2,
SCORES_CLASSES_DIMS);
if (*scores_tensor == NULL)
gint boxesIndex = gst_tensor_meta_get_index_from_id (tensor_meta,
g_quark_from_static_string (GST_MODEL_OBJECT_DETECTOR_BOXES));
gint scoresIndex = gst_tensor_meta_get_index_from_id (tensor_meta,
g_quark_from_static_string (GST_MODEL_OBJECT_DETECTOR_SCORES));
gint numDetectionsIndex = gst_tensor_meta_get_index_from_id (tensor_meta,
g_quark_from_static_string (GST_MODEL_OBJECT_DETECTOR_NUM_DETECTIONS));
gint clasesIndex = gst_tensor_meta_get_index_from_id (tensor_meta,
g_quark_from_static_string (GST_MODEL_OBJECT_DETECTOR_CLASSES));
if (boxesIndex == -1 || scoresIndex == -1 || numDetectionsIndex == -1)
continue;
*numdetect_tensor = gst_tensor_meta_get_typed_tensor (tmeta,
g_quark_from_static_string (GST_MODEL_OBJECT_DETECTOR_NUM_DETECTIONS),
GST_TENSOR_DATA_TYPE_FLOAT32, GST_TENSOR_DIM_ORDER_ROW_MAJOR, 1,
NUM_DETECT_DIMS);
if (*numdetect_tensor == NULL)
*numdetect_tensor = gst_tensor_meta_get_typed_tensor (tmeta,
g_quark_from_static_string (GST_MODEL_OBJECT_DETECTOR_NUM_DETECTIONS),
GST_TENSOR_DATA_TYPE_UINT32, GST_TENSOR_DIM_ORDER_ROW_MAJOR, 1,
NUM_DETECT_DIMS);
if (*numdetect_tensor == NULL)
if (tensor_meta->num_tensors == 4 && clasesIndex == -1)
continue;
*classes_tensor = gst_tensor_meta_get_typed_tensor (tmeta,
g_quark_from_static_string (GST_MODEL_OBJECT_DETECTOR_CLASSES),
GST_TENSOR_DATA_TYPE_FLOAT32, GST_TENSOR_DIM_ORDER_ROW_MAJOR, 2,
SCORES_CLASSES_DIMS);
if (*classes_tensor == NULL)
*classes_tensor = gst_tensor_meta_get_typed_tensor (tmeta,
g_quark_from_static_string (GST_MODEL_OBJECT_DETECTOR_CLASSES),
GST_TENSOR_DATA_TYPE_UINT32, GST_TENSOR_DIM_ORDER_ROW_MAJOR, 2,
SCORES_CLASSES_DIMS);
return TRUE;
return tensor_meta;
}
return FALSE;
return NULL;
}
static gboolean
@ -412,7 +380,7 @@ gst_ssd_object_detector_transform_ip (GstBaseTransform * trans, GstBuffer * buf)
#define DEFINE_GET_FUNC(TYPE, MAX) \
static gboolean \
get_ ## TYPE ## _at_index (const GstTensor *tensor, GstMapInfo *map, \
get_ ## TYPE ## _at_index (GstTensor *tensor, GstMapInfo *map, \
guint index, TYPE * out) \
{ \
switch (tensor->data_type) { \
@ -437,16 +405,18 @@ gst_ssd_object_detector_transform_ip (GstBaseTransform * trans, GstBuffer * buf)
return TRUE; \
}
DEFINE_GET_FUNC (guint32, UINT32_MAX);
DEFINE_GET_FUNC (float, FLOAT_MAX);
DEFINE_GET_FUNC (guint32, UINT32_MAX)
DEFINE_GET_FUNC (float, FLOAT_MAX)
#undef DEFINE_GET_FUNC
static void
extract_bounding_boxes (GstSsdObjectDetector * self, gsize w, gsize h,
GstAnalyticsRelationMeta * rmeta, const GstTensor * classes_tensor,
const GstTensor * numdetect_tensor, const GstTensor * scores_tensor,
const GstTensor * boxes_tensor)
static void
extract_bounding_boxes (GstSsdObjectDetector * self, gsize w, gsize h,
GstAnalyticsRelationMeta * rmeta, GstTensorMeta * tmeta)
{
gint classes_index;
gint boxes_index;
gint scores_index;
gint numdetect_index;
GstMapInfo boxes_map = GST_MAP_INFO_INIT;
GstMapInfo numdetect_map = GST_MAP_INFO_INIT;
GstMapInfo scores_map = GST_MAP_INFO_INIT;
@ -454,49 +424,57 @@ extract_bounding_boxes (GstSsdObjectDetector * self, gsize w, gsize h,
guint num_detections = 0;
if (numdetect_tensor == NULL || scores_tensor == NULL || boxes_tensor == NULL) {
classes_index = gst_tensor_meta_get_index_from_id (tmeta,
g_quark_from_static_string (GST_MODEL_OBJECT_DETECTOR_CLASSES));
numdetect_index = gst_tensor_meta_get_index_from_id (tmeta,
g_quark_from_static_string (GST_MODEL_OBJECT_DETECTOR_NUM_DETECTIONS));
scores_index = gst_tensor_meta_get_index_from_id (tmeta,
g_quark_from_static_string (GST_MODEL_OBJECT_DETECTOR_SCORES));
boxes_index = gst_tensor_meta_get_index_from_id (tmeta,
g_quark_from_static_string (GST_MODEL_OBJECT_DETECTOR_BOXES));
if (numdetect_index == -1 || scores_index == -1 || numdetect_index == -1) {
GST_WARNING ("Missing tensor data expected for SSD model");
return;
}
if (!gst_buffer_map (numdetect_tensor->data, &numdetect_map, GST_MAP_READ)) {
GST_ERROR_OBJECT (self, "Failed to map numdetect tensor memory");
if (!gst_buffer_map (tmeta->tensors[numdetect_index]->data, &numdetect_map,
GST_MAP_READ)) {
GST_ERROR_OBJECT (self, "Failed to map tensor memory for index %d",
numdetect_index);
goto cleanup;
}
if (!gst_buffer_map (boxes_tensor->data, &boxes_map, GST_MAP_READ)) {
GST_ERROR_OBJECT (self, "Failed to map boxes tensor memory");
if (!gst_buffer_map (tmeta->tensors[boxes_index]->data, &boxes_map,
GST_MAP_READ)) {
GST_ERROR_OBJECT (self, "Failed to map tensor memory for index %d",
boxes_index);
goto cleanup;
}
if (!gst_buffer_map (scores_tensor->data, &scores_map, GST_MAP_READ)) {
GST_ERROR_OBJECT (self, "Failed to map scores tensor memory");
if (!gst_buffer_map (tmeta->tensors[scores_index]->data, &scores_map,
GST_MAP_READ)) {
GST_ERROR_OBJECT (self, "Failed to map tensor memory for index %d",
scores_index);
goto cleanup;
}
if (classes_tensor &&
!gst_buffer_map (classes_tensor->data, &classes_map, GST_MAP_READ)) {
GST_DEBUG_OBJECT (self, "Failed to map classes tensor memory");
goto cleanup;
if (classes_index != -1 &&
!gst_buffer_map (tmeta->tensors[classes_index]->data, &classes_map,
GST_MAP_READ)) {
GST_DEBUG_OBJECT (self, "Failed to map tensor memory for index %d",
classes_index);
}
if (!get_guint32_at_index (numdetect_tensor, &numdetect_map,
if (!get_guint32_at_index (tmeta->tensors[numdetect_index], &numdetect_map,
0, &num_detections)) {
GST_ERROR_OBJECT (self, "Failed to get the number of detections");
goto cleanup;
}
GST_LOG_OBJECT (self, "Model claims %u detections", num_detections);
num_detections = MIN (num_detections, scores_tensor->dims[1]);
num_detections = MIN (num_detections, boxes_tensor->dims[1]);
if (classes_tensor)
num_detections = MIN (num_detections, classes_tensor->dims[1]);
GST_LOG_OBJECT (self, "Model really has %u detections"
" (%zu scores, %zu boxes, %zu classes)", num_detections,
scores_tensor->dims[1], boxes_tensor->dims[1],
classes_tensor ? classes_tensor->dims[1] : 0);
GST_LOG_OBJECT (self, "Model claims %d detections", num_detections);
for (int i = 0; i < num_detections; i++) {
float score;
@ -506,20 +484,25 @@ extract_bounding_boxes (GstSsdObjectDetector * self, gsize w, gsize h,
GQuark label = 0;
GstAnalyticsODMtd odmtd;
if (!get_float_at_index (scores_tensor, &scores_map, i, &score))
if (!get_float_at_index (tmeta->tensors[numdetect_index], &scores_map,
i, &score))
continue;
GST_LOG_OBJECT (self, "Detection %u score is %f", i, score);
if (score < self->score_threshold)
continue;
if (!get_float_at_index (boxes_tensor, &boxes_map, i * 4, &y))
if (!get_float_at_index (tmeta->tensors[boxes_index], &boxes_map,
i * 4, &y))
continue;
if (!get_float_at_index (boxes_tensor, &boxes_map, i * 4 + 1, &x))
if (!get_float_at_index (tmeta->tensors[boxes_index], &boxes_map,
i * 4 + 1, &x))
continue;
if (!get_float_at_index (boxes_tensor, &boxes_map, i * 4 + 2, &bheight))
if (!get_float_at_index (tmeta->tensors[boxes_index], &boxes_map,
i * 4 + 2, &bheight))
continue;
if (!get_float_at_index (boxes_tensor, &boxes_map, i * 4 + 3, &bwidth))
if (!get_float_at_index (tmeta->tensors[boxes_index], &boxes_map,
i * 4 + 3, &bwidth))
continue;
if (CLAMP (bwidth, 0, 1) * CLAMP (bheight, 0, 1) > self->size_threshold) {
@ -530,7 +513,8 @@ extract_bounding_boxes (GstSsdObjectDetector * self, gsize w, gsize h,
}
if (self->labels && classes_map.memory &&
get_guint32_at_index (classes_tensor, &classes_map, i, &bclass)) {
get_guint32_at_index (tmeta->tensors[classes_index], &classes_map,
i, &bclass)) {
if (bclass < self->labels->len)
label = g_array_index (self->labels, GQuark, bclass);
}
@ -552,13 +536,13 @@ extract_bounding_boxes (GstSsdObjectDetector * self, gsize w, gsize h,
cleanup:
if (numdetect_map.memory)
gst_buffer_unmap (numdetect_tensor->data, &numdetect_map);
gst_buffer_unmap (tmeta->tensors[numdetect_index]->data, &numdetect_map);
if (classes_map.memory)
gst_buffer_unmap (classes_tensor->data, &classes_map);
gst_buffer_unmap (tmeta->tensors[classes_index]->data, &classes_map);
if (scores_map.memory)
gst_buffer_unmap (scores_tensor->data, &scores_map);
gst_buffer_unmap (tmeta->tensors[scores_index]->data, &scores_map);
if (boxes_map.memory)
gst_buffer_unmap (boxes_tensor->data, &boxes_map);
gst_buffer_unmap (tmeta->tensors[boxes_index]->data, &boxes_map);
}
@ -566,15 +550,12 @@ static gboolean
gst_ssd_object_detector_process (GstBaseTransform * trans, GstBuffer * buf)
{
GstSsdObjectDetector *self = GST_SSD_OBJECT_DETECTOR (trans);
GstTensorMeta *tmeta;
GstAnalyticsRelationMeta *rmeta;
const GstTensor *classes_tensor = NULL;
const GstTensor *numdetect_tensor = NULL;
const GstTensor *scores_tensor = NULL;
const GstTensor *boxes_tensor = NULL;
// get all tensor metas
if (!gst_ssd_object_detector_get_tensors (self, buf,
&classes_tensor, &numdetect_tensor, &scores_tensor, &boxes_tensor)) {
tmeta = gst_ssd_object_detector_get_tensor_meta (self, buf);
if (!tmeta) {
GST_WARNING_OBJECT (trans, "missing tensor meta");
return TRUE;
} else {
@ -583,8 +564,7 @@ gst_ssd_object_detector_process (GstBaseTransform * trans, GstBuffer * buf)
}
extract_bounding_boxes (self, self->video_info.width,
self->video_info.height, rmeta, classes_tensor, numdetect_tensor,
scores_tensor, boxes_tensor);
self->video_info.height, rmeta, tmeta);
return TRUE;
}

5
subprojects/gst-plugins-bad/gst/tensordecoders/gsttensordecoders.c
View File

@ -26,8 +26,6 @@
#include "gstssdobjectdetector.h"
#include "gstyolotensordecoder.h"
#include "gstclassifiertensordecoder.h"
#include "gstfacedetectortensordecoder.h"
/**
* SECTION:plugin-tensordecoders
@ -43,9 +41,6 @@ plugin_init (GstPlugin * plugin)
ret |= GST_ELEMENT_REGISTER (ssd_object_detector, plugin);
ret |= GST_ELEMENT_REGISTER (yolo_seg_tensor_decoder, plugin);
ret |= GST_ELEMENT_REGISTER (yolo_od_tensor_decoder, plugin);
ret |= GST_ELEMENT_REGISTER (yolo_tensor_decoder, plugin);
ret |= GST_ELEMENT_REGISTER (classifier_tensor_decoder, plugin);
ret |= GST_ELEMENT_REGISTER (face_detector_tensor_decoder, plugin);
return ret;
}