From c941ded4ba0c21bdc6fa9ef89b8cb383124483c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Olivier=20Cr=C3=AAte?= Date: Sun, 6 Apr 2025 12:05:48 -0400 Subject: [PATCH] tflite: Add Coral EdgeTPU inference element Part-of: --- .../gst-plugins-bad/ext/tflite/gsttflite.c | 14 +- .../ext/tflite/gsttfliteedgetpuinference.c | 165 ++++++++++++++++++ .../ext/tflite/gsttfliteedgetpuinference.h | 39 +++++ .../ext/tflite/gsttfliteinference.c | 6 +- .../ext/tflite/gsttfliteinference.h | 5 + .../gst-plugins-bad/ext/tflite/meson.build | 25 ++- subprojects/gst-plugins-bad/meson.options | 1 + 7 files changed, 248 insertions(+), 7 deletions(-) create mode 100644 subprojects/gst-plugins-bad/ext/tflite/gsttfliteedgetpuinference.c create mode 100644 subprojects/gst-plugins-bad/ext/tflite/gsttfliteedgetpuinference.h diff --git a/subprojects/gst-plugins-bad/ext/tflite/gsttflite.c b/subprojects/gst-plugins-bad/ext/tflite/gsttflite.c index 2acfeb09fe..182dd0e9b0 100644 --- a/subprojects/gst-plugins-bad/ext/tflite/gsttflite.c +++ b/subprojects/gst-plugins-bad/ext/tflite/gsttflite.c @@ -26,10 +26,20 @@ #include "gsttfliteinference.h" +#ifdef EDGETPU +#include "gsttfliteedgetpuinference.h" +#endif + static gboolean -plugin_init (GstPlugin *plugin) +plugin_init (GstPlugin * plugin) { - return GST_ELEMENT_REGISTER (tflite_inference, plugin); + gboolean ret = GST_ELEMENT_REGISTER (tflite_inference, plugin); + +#ifdef EDGETPU + ret |= GST_ELEMENT_REGISTER (tflite_edgetpu_inference, plugin); +#endif + + return ret; } GST_PLUGIN_DEFINE (GST_VERSION_MAJOR, diff --git a/subprojects/gst-plugins-bad/ext/tflite/gsttfliteedgetpuinference.c b/subprojects/gst-plugins-bad/ext/tflite/gsttfliteedgetpuinference.c new file mode 100644 index 0000000000..40a8bb7674 --- /dev/null +++ b/subprojects/gst-plugins-bad/ext/tflite/gsttfliteedgetpuinference.c @@ -0,0 +1,165 @@ +/* + * GStreamer + * Copyright (C) 2025 Collabora Ltd. + * + * gsttfliteedgetpuinference.c + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +/** + * SECTION:element-tfliteedgetpuinference + * @short_description: Run TFLITE inference model on video buffers using a EdgeTpu device + * + * This element can apply an TFLITE model to video buffers. It attaches + * the tensor output to the buffer as a @ref GstTensorMeta. + * + * Uses the Google Coral EdgeTpu devices. + * + * To install TFLITE on your system, follow the instructions in the + * README.md in with this plugin. + * + * ## Example launch command: + * + * GST_DEBUG=ssdobjectdetector:5 \ + * gst-launch-1.0 filesrc location=tflite-models/images/bus.jpg ! \ + * jpegdec ! videoconvert ! tfliteedgetpuinference model-file=tflite-models/models/ssd_mobilenet_v1_coco.tflite ! \ + * ssdobjectdetector label-file=tflite-models/labels/COCO_classes.txt ! videoconvert ! imagefreeze ! autovideosink + * + */ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "gsttfliteedgetpuinference.h" +#include + + +typedef struct _GstTFliteEdgeTpuInference +{ + GstTFliteInference parent; + + TfLiteDelegate *tflite_delegate; +} GstTFliteEdgeTpuInference; + +GST_DEBUG_CATEGORY (tflite_edgetpu_inference_debug); +#define GST_CAT_DEFAULT tflite_edgetpu_inference_debug + +GST_ELEMENT_REGISTER_DEFINE (tflite_edgetpu_inference, + "tfliteedgetpuinference", GST_RANK_NONE, GST_TYPE_TFLITE_EDGETPU_INFERENCE); + + +static gboolean gst_tflite_edgetpu_update_options (GstTFliteInference * inf, + TfLiteInterpreterOptions * interpreter_options); +static gboolean gst_tflite_edgetpu_inference_stop (GstBaseTransform * trans); + +G_DEFINE_TYPE (GstTFliteEdgeTpuInference, gst_tflite_edgetpu_inference, + GST_TYPE_TFLITE_INFERENCE); + +static void +gst_tflite_edgetpu_inference_class_init (GstTFliteEdgeTpuInferenceClass * klass) +{ + GstElementClass *element_class = (GstElementClass *) klass; + GstBaseTransformClass *basetransform_class = (GstBaseTransformClass *) klass; + GstTFliteInferenceClass *tflite_class = (GstTFliteInferenceClass *) klass; + + GST_DEBUG_CATEGORY_INIT (tflite_edgetpu_inference_debug, + "tfliteedgetpuinference", 0, "TFLlite edgetpu inference"); + + gst_element_class_set_static_metadata (element_class, + "tfliteedgetpuinference", + "Filter/Effect", + "Apply neural network to video frames and create tensor output" + " using the Google Edge TPU", + "Olivier Crête "); + + basetransform_class->stop = gst_tflite_edgetpu_inference_stop; + + tflite_class->update_options = gst_tflite_edgetpu_update_options; +} + +static void +gst_tflite_edgetpu_inference_init (GstTFliteEdgeTpuInference * self) +{ +} + +static gboolean +gst_tflite_edgetpu_update_options (GstTFliteInference * inf, + TfLiteInterpreterOptions * interpreter_options) +{ + GstTFliteEdgeTpuInference *self = GST_TFLITE_EDGETPU_INFERENCE (inf); + size_t num_devices = 0; + struct edgetpu_device *devices; + + devices = edgetpu_list_devices (&num_devices); + + if (num_devices == 0) { + GST_ERROR_OBJECT (self, + "Could not create EdgeTPU session because no EdgeTPU" + " device is connected"); + return FALSE; + } + + /* Not passing options or a path for now */ + self->tflite_delegate = edgetpu_create_delegate (devices[0].type, + devices[0].path, NULL, 0); + + if (self->tflite_delegate == NULL) { + GST_ERROR_OBJECT (self, "Could not create EdgeTPU session"); + edgetpu_free_devices (devices); + return FALSE; + } + + const gchar *dev_type_str = ""; + switch (devices[0].type) { + case EDGETPU_APEX_PCI: + dev_type_str = "PCIe"; + break; + case EDGETPU_APEX_USB: + dev_type_str = "USB"; + break; + default: + dev_type_str = "unknown"; + break; + } + + GST_DEBUG ("Using EdgeTPU version %s device of type %s at %s", + edgetpu_version (), dev_type_str, devices[0].path); + + edgetpu_free_devices (devices); + + if (self->tflite_delegate) + TfLiteInterpreterOptionsAddDelegate (interpreter_options, + self->tflite_delegate); + + return TRUE; +} + +static gboolean +gst_tflite_edgetpu_inference_stop (GstBaseTransform * trans) +{ + GstTFliteEdgeTpuInference *self = GST_TFLITE_EDGETPU_INFERENCE (trans); + gboolean ret; + + ret = GST_BASE_TRANSFORM_CLASS (gst_tflite_edgetpu_inference_parent_class) + ->stop (trans); + + if (self->tflite_delegate) + edgetpu_free_delegate (self->tflite_delegate); + self->tflite_delegate = NULL; + + return ret; +} diff --git a/subprojects/gst-plugins-bad/ext/tflite/gsttfliteedgetpuinference.h b/subprojects/gst-plugins-bad/ext/tflite/gsttfliteedgetpuinference.h new file mode 100644 index 0000000000..b0e036a852 --- /dev/null +++ b/subprojects/gst-plugins-bad/ext/tflite/gsttfliteedgetpuinference.h @@ -0,0 +1,39 @@ +/* + * GStreamer gstreamer-tfliteinference + * Copyright (C) 2024 Collabora Ltd + * + * gsttfliteinference.h + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, + * Boston, MA 02110-1301, USA. + */ + +#ifndef __GST_TFLITE_EDGETPU_INFERENCE_H__ +#define __GST_TFLITE_EDGETPU_INFERENCE_H__ + +#include "gsttfliteinference.h" + + +G_BEGIN_DECLS + +#define GST_TYPE_TFLITE_EDGETPU_INFERENCE (gst_tflite_edgetpu_inference_get_type()) +G_DECLARE_FINAL_TYPE (GstTFliteEdgeTpuInference, gst_tflite_edgetpu_inference, GST, + TFLITE_EDGETPU_INFERENCE, GstTFliteInference) + +GST_ELEMENT_REGISTER_DECLARE (tflite_edgetpu_inference) + +G_END_DECLS + +#endif /* __GST_TFLITE_INFERENCE_H__ */ diff --git a/subprojects/gst-plugins-bad/ext/tflite/gsttfliteinference.c b/subprojects/gst-plugins-bad/ext/tflite/gsttfliteinference.c index 335ad8e860..667d1220ce 100644 --- a/subprojects/gst-plugins-bad/ext/tflite/gsttfliteinference.c +++ b/subprojects/gst-plugins-bad/ext/tflite/gsttfliteinference.c @@ -42,8 +42,6 @@ #include "config.h" #endif -#include "tensorflow/lite/c/c_api.h" - #include #include #include "gsttfliteinference.h" @@ -511,6 +509,10 @@ gst_tflite_inference_start (GstBaseTransform * trans) priv->numberOfThreads); } + if (klass->update_options) + if (!klass->update_options (self, priv->interpreter_options)) + goto error; + priv->interpreter = TfLiteInterpreterCreate (priv->model, priv->interpreter_options); if (!priv->interpreter) { diff --git a/subprojects/gst-plugins-bad/ext/tflite/gsttfliteinference.h b/subprojects/gst-plugins-bad/ext/tflite/gsttfliteinference.h index a6896ccc43..fe3cfcc634 100644 --- a/subprojects/gst-plugins-bad/ext/tflite/gsttfliteinference.h +++ b/subprojects/gst-plugins-bad/ext/tflite/gsttfliteinference.h @@ -26,6 +26,8 @@ #include #include +#include "tensorflow/lite/c/c_api.h" + G_BEGIN_DECLS #define GST_TYPE_TFLITE_INFERENCE (gst_tflite_inference_get_type()) @@ -36,6 +38,9 @@ GST_ELEMENT_REGISTER_DECLARE (tflite_inference) struct _GstTFliteInferenceClass { GstBaseTransformClass basetransform; + + gboolean (*update_options) (GstTFliteInference * self, + TfLiteInterpreterOptions * interpreter_options); }; G_END_DECLS diff --git a/subprojects/gst-plugins-bad/ext/tflite/meson.build b/subprojects/gst-plugins-bad/ext/tflite/meson.build index 52efd93b58..549ac1dae4 100644 --- a/subprojects/gst-plugins-bad/ext/tflite/meson.build +++ b/subprojects/gst-plugins-bad/ext/tflite/meson.build @@ -5,11 +5,16 @@ tflite_sources = [ ] tflite_headers = [ - 'gstfliteinference.h' + 'gsttfliteinference.h', + 'gsttfliteedgetpuinference.h' +] + +edgetpu_sources = [ + 'gsttfliteedgetpuinference.c' ] doc_sources = [] -foreach s: tflite_sources + tflite_headers +foreach s: tflite_sources + tflite_headers + edgetpu_sources doc_sources += meson.current_source_dir() / s endforeach @@ -29,6 +34,7 @@ tensorflow_lite_header_found = cc.has_header('tensorflow/lite/c/c_api.h', required: get_option('tflite')) if tensorflow_lite_dep.found() and tensorflow_lite_header_found + tflite_extra_dep = [] tflite_c_args = [] if cc.has_header_symbol('tensorflow/lite/c/c_api.h', 'kTfLiteBFloat16', @@ -36,12 +42,25 @@ if tensorflow_lite_dep.found() and tensorflow_lite_header_found tflite_c_args += ['-DTFLITE_HAS_BFLOAT16'] endif + edgetpu_dep = cc.find_library('edgetpu', + required : get_option('tflite-edgetpu')) + + if edgetpu_dep.found() and cc.has_header('libedgetpu/edgetpu_c.h', + dependencies: edgetpu_dep, + required: get_option('tflite-edgetpu')) + tflite_c_args += ['-DEDGETPU','-DTFLITE_USE_OPAQUE_DELEGATE=0', + '-DTFLITE_WITH_STABLE_ABI=0'] + tflite_sources += edgetpu_sources + tflite_extra_dep += [edgetpu_dep] + endif + + gsttflite = library('gsttflite', tflite_sources, c_args : gst_plugins_bad_args + tflite_c_args, include_directories : [configinc, libsinc], dependencies : [gstbase_dep, gstvideo_dep, gstanalytics_dep, - tensorflow_lite_dep,libm, gio_dep], + tensorflow_lite_dep,libm, gio_dep, tflite_extra_dep], install : true, install_dir : plugins_install_dir, ) diff --git a/subprojects/gst-plugins-bad/meson.options b/subprojects/gst-plugins-bad/meson.options index ae3cd064c6..27e65cf084 100644 --- a/subprojects/gst-plugins-bad/meson.options +++ b/subprojects/gst-plugins-bad/meson.options @@ -177,6 +177,7 @@ option('svthevcenc', type : 'feature', value : 'auto', description : 'Scalable V option('svtjpegxs', type : 'feature', value : 'auto', description : 'Scalable Video Technology for JPEG-XS plugin') option('teletext', type : 'feature', value : 'auto', description : 'Teletext plugin') option('tflite', type : 'feature', value : 'auto', description : 'TensorFlow Lite (LiteRT) plugin') +option('tflite-edgetpu', type : 'feature', value : 'auto', description : 'TensorFlow Lite (LiteRT) EdgeTPU (Coral) support') option('tinyalsa', type : 'feature', value : 'auto', description : 'TinyALSA plugin') option('transcode', type : 'feature', value : 'auto', description : 'Transcode plugin') option('ttml', type : 'feature', value : 'auto', description : 'TTML subtitle parser and renderer plugin')