FaSTTUBe/GST-Tensordecoder-ov_ep
37
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

analytics: add a convenient API to retrieve tensor

Browse Source 
use the API in facedetector tensor decoding

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/9162>
This commit is contained in:
raghu447 2025-06-02 16:00:36 +05:30
parent a5586a50c2
commit 4b6c732fdd
6 changed files with 205 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

73
girs/GstAnalytics-1.0.gir
View File

@ -1338,6 +1338,39 @@ dimension is dynamic.</doc>
</parameter>
</parameters>
</constructor>
<method name="check_type" c:identifier="gst_tensor_check_type" version="1.28">
<doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gsttensor.c">Validate the tensor whether it mathces the reading order, dimensions and the data type.
Validate whether the #GstBuffer has enough size to hold the tensor data.</doc>
<source-position filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gsttensor.h"/>
<return-value transfer-ownership="none">
<doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gsttensor.c">TRUE if the #GstTensor has the reading order from the memory matching @order,
dimensions matching @num_dims, data type matching @data_type and the #GstBuffer mathcing @data
has enough size to hold the tensor data.
Otherwise FALSE will be returned.</doc>
<type name="gboolean" c:type="gboolean"/>
</return-value>
<parameters>
<instance-parameter name="tensor" transfer-ownership="none">
<type name="Tensor" c:type="const GstTensor*"/>
</instance-parameter>
<parameter name="order" transfer-ownership="none">
<doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gsttensor.c">The order of the tensor to read from the memory</doc>
<type name="TensorDimOrder" c:type="GstTensorDimOrder"/>
</parameter>
<parameter name="num_dims" transfer-ownership="none">
<doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gsttensor.c">The number of dimensions that the tensor can have</doc>
<type name="gsize" c:type="gsize"/>
</parameter>
<parameter name="data_type" transfer-ownership="none">
<doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gsttensor.c">The data type of the tensor</doc>
<type name="TensorDataType" c:type="GstTensorDataType"/>
</parameter>
<parameter name="data" transfer-ownership="full">
<doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gsttensor.c">#GstBuffer holding tensor data</doc>
<type name="Gst.Buffer" c:type="GstBuffer*"/>
</parameter>
</parameters>
</method>
<method name="copy" c:identifier="gst_tensor_copy" version="1.26">
<doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gsttensor.c">Create a copy of @tensor.</doc>
<source-position filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gsttensor.h"/>
@ -1533,6 +1566,46 @@ Otherwise NULL will be returned.</doc>
</parameter>
</parameters>
</method>
<method name="get_typed_tensor" c:identifier="gst_tensor_meta_get_typed_tensor" version="1.28">
<doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gsttensormeta.c">Get the first tensor from the #GstTensorMeta identified by @tensor_id, mathcing
the reading order, dimensions and the data type.
Validate whether the #GstBuffer has enough size to hold the tensor data.</doc>
<source-position filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gsttensormeta.h"/>
<return-value transfer-ownership="none" nullable="1">
<doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gsttensormeta.c">a GstTensor with id matching @tensor_id,
reading order from the memory matching @order, dimensions matching @num_dims,
data type matching @data_type. The #GstBuffer mathcing @data should
have enough size to hold the tensor data.
Otherwise NULL will be returned.</doc>
<type name="Tensor" c:type="const GstTensor*"/>
</return-value>
<parameters>
<instance-parameter name="tmeta" transfer-ownership="none">
<doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gsttensormeta.c">A #GstTensorMeta</doc>
<type name="TensorMeta" c:type="GstTensorMeta*"/>
</instance-parameter>
<parameter name="tensor_id" transfer-ownership="none">
<doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gsttensormeta.c">A #GQuark identifying the tensor-encoding</doc>
<type name="GLib.Quark" c:type="GQuark"/>
</parameter>
<parameter name="order" transfer-ownership="none">
<doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gsttensormeta.c">The order of the tensor to read from the memory</doc>
<type name="TensorDimOrder" c:type="GstTensorDimOrder"/>
</parameter>
<parameter name="num_dims" transfer-ownership="none">
<doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gsttensormeta.c">The number of dimensions that the tensor can have</doc>
<type name="gsize" c:type="gsize"/>
</parameter>
<parameter name="data_type" transfer-ownership="none">
<doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gsttensormeta.c">The data type of the tensor</doc>
<type name="TensorDataType" c:type="GstTensorDataType"/>
</parameter>
<parameter name="data" transfer-ownership="full">
<doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gsttensormeta.c">#GstBuffer holding tensor data</doc>
<type name="Gst.Buffer" c:type="GstBuffer*"/>
</parameter>
</parameters>
</method>
<method name="set" c:identifier="gst_tensor_meta_set" version="1.26">
<doc xml:space="preserve" filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gsttensormeta.c">Sets tensors into the #GstTensorMeta</doc>
<source-position filename="../subprojects/gst-plugins-bad/gst-libs/gst/analytics/gsttensormeta.h"/>

58
subprojects/gst-plugins-bad/gst-libs/gst/analytics/gsttensor.c
View File

@ -256,3 +256,61 @@ gst_tensor_data_type_get_name (GstTensorDataType data_type)
return NULL;
}
}
/**
* gst_tensor_check_type:
* @tmeta: A #GstTensor
* @order: The order of the tensor to read from the memory
* @num_dims: The number of dimensions that the tensor can have
* @data_type: The data type of the tensor
* @data: (transfer full): #GstBuffer holding tensor data
*
* Validate the tensor whether it mathces the reading order, dimensions and the data type.
* Validate whether the #GstBuffer has enough size to hold the tensor data.
*
* Returns: TRUE if the #GstTensor has the reading order from the memory matching @order,
* dimensions matching @num_dims, data type matching @data_type and the #GstBuffer mathcing @data
* has enough size to hold the tensor data.
* Otherwise FALSE will be returned.
*
* Since: 1.28
*/
gboolean
gst_tensor_check_type (const GstTensor * tensor, GstTensorDimOrder order,
gsize num_dims, GstTensorDataType data_type, GstBuffer * data)
{
gsize num_elements = 1, tensor_size, i;
if (tensor->dims_order != order) {
GST_DEBUG ("Tensor has order %d, expected %d", tensor->dims_order, order);
return FALSE;
}
if (tensor->num_dims != num_dims) {
GST_DEBUG ("Tensor has %zu dimensions, expected %zu", tensor->num_dims,
num_dims);
return FALSE;
}
if (tensor->data_type != data_type) {
GST_DEBUG ("Tensor has data type \"%s\", expected \"%s\".",
gst_tensor_data_type_get_name (tensor->data_type),
gst_tensor_data_type_get_name (data_type));
return FALSE;
}
for (i = 0; i < tensor->num_dims; i++) {
num_elements *= tensor->dims[i];
}
tensor_size = size_for_elements (tensor->data_type, num_elements);
if (gst_buffer_get_size (data) < tensor_size) {
GST_DEBUG ("Expected buffer of size %zu (%zu elements),"
" but buffer has size %zu", tensor_size, num_elements,
gst_buffer_get_size (data));
return FALSE;
}
return TRUE;
}

4
subprojects/gst-plugins-bad/gst-libs/gst/analytics/gsttensor.h
View File

@ -152,6 +152,10 @@ GType gst_tensor_get_type (void);
GST_ANALYTICS_META_API
const gchar *gst_tensor_data_type_get_name (GstTensorDataType data_type);
GST_ANALYTICS_META_API
gboolean gst_tensor_check_type(const GstTensor * tensor, GstTensorDimOrder order,
gsize num_dims, GstTensorDataType data_type, GstBuffer * data);
G_END_DECLS
#endif /* __GST_TENSOR_H__ */

37
subprojects/gst-plugins-bad/gst-libs/gst/analytics/gsttensormeta.c
View File

@ -206,6 +206,43 @@ gst_tensor_meta_get_by_id (GstTensorMeta * tmeta, GQuark id)
return NULL;
}
/**
* gst_tensor_meta_get_typed_tensor:
* @tmeta: A #GstTensorMeta
* @tensor_id: A #GQuark identifying the tensor-encoding
* @order: The order of the tensor to read from the memory
* @num_dims: The number of dimensions that the tensor can have
* @data_type: The data type of the tensor
* @data: (transfer full): #GstBuffer holding tensor data
*
* Get the first tensor from the #GstTensorMeta identified by @tensor_id, mathcing
* the reading order, dimensions and the data type.
* Validate whether the #GstBuffer has enough size to hold the tensor data.
*
* Return: (nullable)(transfer none): a GstTensor with id matching @tensor_id,
* reading order from the memory matching @order, dimensions matching @num_dims,
* data type matching @data_type. The #GstBuffer mathcing @data should
* have enough size to hold the tensor data.
* Otherwise NULL will be returned.
*
* Since: 1.28
*/
const GstTensor *
gst_tensor_meta_get_typed_tensor (GstTensorMeta * tmeta,
GQuark tensor_id, GstTensorDimOrder order, gsize num_dims,
GstTensorDataType data_type, GstBuffer * data)
{
const GstTensor *tensor;
tensor = gst_tensor_meta_get_by_id (tmeta, tensor_id);
if (!gst_tensor_check_type (tensor, order, num_dims, data_type, data)) {
return NULL;
}
return tensor;
}
/**
* gst_tensor_meta_get:
* @tmeta: A #GstTensorMeta

4
subprojects/gst-plugins-bad/gst-libs/gst/analytics/gsttensormeta.h
View File

@ -82,6 +82,10 @@ void gst_tensor_meta_set (GstTensorMeta *tmeta, guint num_tensors,
GST_ANALYTICS_META_API
const GstTensor *gst_tensor_meta_get_by_id (GstTensorMeta *tmeta, GQuark id);
GST_ANALYTICS_META_API
const GstTensor *gst_tensor_meta_get_typed_tensor (GstTensorMeta * tmeta,
GQuark tensor_id, GstTensorDimOrder order, gsize num_dims, GstTensorDataType data_type, GstBuffer * data);
GST_ANALYTICS_META_API
const GstTensor *gst_tensor_meta_get (GstTensorMeta *tmeta, gsize index);

80
subprojects/gst-plugins-bad/gst/tensordecoders/gstfacedetectortensordecoder.c
View File

@ -294,16 +294,18 @@ gst_face_detector_tensor_decoder_set_caps (GstBaseTransform * trans,
* @buf:in: buffer
* @boxes_tensor:out: Boxes tensor
* @scores_tensor:out: scores tensor
* @return: TRUE if buf has boxes and scores tensor attach to it.
*
* Retrieve FaceDetection boxes and scores tensors from buffer.
*
* @return: TRUE if buf has boxes and scores tensor with desired features are attached to it.
* Otherwise FALSE will be returned.
*/
static gboolean
gst_face_detector_tensor_decoder_get_tensor_meta (GstFaceDetectorTensorDecoder
* self, GstBuffer * buf, GstTensor ** boxes_tensor,
GstTensor ** scores_tensor)
* self, GstBuffer * buf, const GstTensor ** boxes_tensor,
const GstTensor ** scores_tensor)
{
GstTensorMeta *tensor_meta;
gint boxes_tensor_idx, scores_tensor_idx;
g_return_val_if_fail (boxes_tensor != NULL, FALSE);
g_return_val_if_fail (scores_tensor != NULL, FALSE);
@ -320,29 +322,26 @@ gst_face_detector_tensor_decoder_get_tensor_meta (GstFaceDetectorTensorDecoder
GST_LOG_OBJECT (self, "Num tensors %zu", tensor_meta->num_tensors);
/* Retrieve the index of the tensor that has a tensor-id matching
* BOXES_TENSOR_ID_QUARK in the GstTensorMeta. */
boxes_tensor_idx = gst_tensor_meta_get_index_from_id (tensor_meta,
BOXES_TENSOR_ID_QUARK);
/* Retrieve the tensor that has a tensor-id matching
* BOXES_TENSOR_ID_QUARK in the GstTensorMeta along with
* the reading order from the memory matching with GST_TENSOR_DIM_ORDER_ROW_MAJOR,
* 3 dimensions and the data type matching with GST_TENSOR_DATA_TYPE_FLOAT32 */
*boxes_tensor =
gst_tensor_meta_get_typed_tensor (tensor_meta, BOXES_TENSOR_ID_QUARK,
GST_TENSOR_DIM_ORDER_ROW_MAJOR, 3, GST_TENSOR_DATA_TYPE_FLOAT32, buf);
/* Retrieve the index of the tensor that has a tensor-id matching*
* SCORES_TENSOR_ID_QUARK in the GstTensorMeta. */
scores_tensor_idx =
gst_tensor_meta_get_index_from_id (tensor_meta, SCORES_TENSOR_ID_QUARK);
/* Retrieve the tensor that has a tensor-id matching
* SCORES_TENSOR_ID_QUARK in the GstTensorMeta along with
* the reading order from the memory matching with GST_TENSOR_DIM_ORDER_ROW_MAJOR,
* 3 dimensions and the data type matching with GST_TENSOR_DATA_TYPE_FLOAT32 */
*scores_tensor =
gst_tensor_meta_get_typed_tensor (tensor_meta, SCORES_TENSOR_ID_QUARK,
GST_TENSOR_DIM_ORDER_ROW_MAJOR, 3, GST_TENSOR_DATA_TYPE_FLOAT32, buf);
if (boxes_tensor_idx >= 0 && scores_tensor_idx >= 0) {
GST_LOG_OBJECT (self, "Boxes tensor id: %d", boxes_tensor_idx);
GST_LOG_OBJECT (self, "Scores tensor id: %d", scores_tensor_idx);
if (*boxes_tensor == NULL || *scores_tensor == NULL)
return FALSE;
*boxes_tensor = tensor_meta->tensors[boxes_tensor_idx];
*scores_tensor = tensor_meta->tensors[scores_tensor_idx];
return TRUE;
} else {
GST_INFO_OBJECT (self, "Couldn't find boxes or scores tensor, skipping");
}
return FALSE;
return TRUE;
}
/* Compare c1 and c2
@ -472,7 +471,7 @@ hard_nms (const GPtrArray * sel_candidates,
*/
static void
gst_face_detector_tensor_decoder_decode_boxes_f32 (GstFaceDetectorTensorDecoder
* self, GstTensor * boxes_tensor, GstTensor * scores_tensor,
* self, const GstTensor * boxes_tensor, const GstTensor * scores_tensor,
GstAnalyticsRelationMeta * rmeta)
{
GstMapInfo map_info_boxes, map_info_scores;
@ -480,9 +479,6 @@ gst_face_detector_tensor_decoder_decode_boxes_f32 (GstFaceDetectorTensorDecoder
gboolean rv;
GPtrArray *sel_candidates = self->sel_candidates, *selected = self->selected;
/* Retrieve memory at index 0 from boxes_tensor in READ mode */
boxes_tensor->data = gst_buffer_make_writable (boxes_tensor->data);
rv = gst_buffer_map (boxes_tensor->data, &map_info_boxes, GST_MAP_READ);
g_assert (rv);
@ -638,33 +634,15 @@ gst_face_detector_tensor_decoder_transform_ip (GstBaseTransform * trans,
GstBuffer * buf)
{
GstFaceDetectorTensorDecoder *self = GST_FACE_DETECTOR_TENSOR_DECODER (trans);
GstTensor *boxes_tensor, *scores_tensor;
const GstTensor *boxes_tensor, *scores_tensor;
GstAnalyticsRelationMeta *rmeta;
/* Retrive the desired Face Detection tensors.
* Return Flow Error if the desired tensors were not supported. */
if (!gst_face_detector_tensor_decoder_get_tensor_meta (self, buf,
&boxes_tensor, &scores_tensor))
return GST_FLOW_OK;
if (boxes_tensor->num_dims != 3) {
&boxes_tensor, &scores_tensor)) {
GST_ELEMENT_ERROR (self, STREAM, DECODE, (NULL),
("Boxes tensor must have 3 dimensions but has %zu",
boxes_tensor->num_dims));
return GST_FLOW_ERROR;
}
if (scores_tensor->num_dims != 3) {
GST_ELEMENT_ERROR (self, STREAM, DECODE, (NULL),
("scores tensor must have 3 dimensions but has %zu",
boxes_tensor->num_dims));
return GST_FLOW_ERROR;
}
if (boxes_tensor->data_type != GST_TENSOR_DATA_TYPE_FLOAT32 &&
scores_tensor->data_type != GST_TENSOR_DATA_TYPE_FLOAT32) {
GST_ELEMENT_ERROR (self, STREAM, NOT_IMPLEMENTED,
("Only data-type FLOAT32 support is implemented"),
("Please implement."));
("Tensor doens't have the expected data type or shape."));
return GST_FLOW_ERROR;
}