wpe: Add software rendering support support

Starting from WPEBackend-FDO 1.6.x, software rendering support is available.
This features allows wpesrc to be used on machines without GPU, and/or for
testing purpose. To enable it, set the `LIBGL_ALWAYS_SOFTWARE=true` environment
variable and make sure `video/x-raw, format=BGRA` caps are negotiated by the
wpesrc element.

ext/wpe/WPEThreadedView.cpp

@@ -26,6 +26,7 @@
 #include <gst/gl/gl.h>
 #include <gst/gl/egl/gsteglimage.h>
 #include <gst/gl/egl/gstgldisplay_egl.h>
+#include <wayland-server.h>
 
 #include <cstdio>
 #include <mutex>
@@ -64,7 +65,7 @@ WPEThreadedView::WPEThreadedView()
     g_mutex_init(&threading.ready_mutex);
     g_cond_init(&threading.ready_cond);
 
-    g_mutex_init(&images.mutex);
+    g_mutex_init(&images_mutex);
 
     {
         GMutexHolder lock(threading.mutex);
@@ -78,15 +79,15 @@ WPEThreadedView::WPEThreadedView()
 WPEThreadedView::~WPEThreadedView()
 {
     {
-        GMutexHolder lock(images.mutex);
+        GMutexHolder lock(images_mutex);
 
-        if (images.pending) {
+        if (egl.pending) {
-            gst_egl_image_unref(images.pending);
+            gst_egl_image_unref(egl.pending);
-            images.pending = nullptr;
+            egl.pending = nullptr;
         }
-        if (images.committed) {
+        if (egl.committed) {
-            gst_egl_image_unref(images.committed);
+            gst_egl_image_unref(egl.committed);
-            images.committed = nullptr;
+            egl.committed = nullptr;
         }
     }
 
@@ -114,7 +115,7 @@ WPEThreadedView::~WPEThreadedView()
     g_cond_clear(&threading.cond);
     g_mutex_clear(&threading.ready_mutex);
     g_cond_clear(&threading.ready_cond);
-    g_mutex_clear(&images.mutex);
+    g_mutex_clear(&images_mutex);
 }
 
 gpointer WPEThreadedView::s_viewThread(gpointer data)
@@ -186,8 +187,18 @@ bool WPEThreadedView::initialize(GstWpeSrc* src, GstGLContext* context, GstGLDis
 #endif
         });
 
-    EGLDisplay eglDisplay = gst_gl_display_egl_get_from_native(GST_GL_DISPLAY_TYPE_WAYLAND,
+    EGLDisplay eglDisplay;
-        gst_gl_display_get_handle(display));
+    if (context && display)
+      eglDisplay = gst_gl_display_egl_get_from_native(GST_GL_DISPLAY_TYPE_WAYLAND,
+                                                      gst_gl_display_get_handle(display));
+    else {
+      eglDisplay = eglGetDisplay(EGL_DEFAULT_DISPLAY);
+      if (eglDisplay == EGL_NO_DISPLAY)
+        return false;
+
+      if (!eglInitialize(eglDisplay, nullptr, nullptr) || !eglBindAPI(EGL_OPENGL_ES_API))
+        return false;
+    }
     GST_DEBUG("eglDisplay %p", eglDisplay);
 
     struct InitializeContext {
@@ -210,8 +221,10 @@ bool WPEThreadedView::initialize(GstWpeSrc* src, GstGLContext* context, GstGLDis
 
             GMutexHolder lock(view.threading.mutex);
 
-            view.gst.context = GST_GL_CONTEXT(gst_object_ref(initializeContext.context));
+            if (initializeContext.context)
-            view.gst.display = GST_GL_DISPLAY(gst_object_ref(initializeContext.display));
+              view.gst.context = GST_GL_CONTEXT(gst_object_ref(initializeContext.context));
+            if (initializeContext.display)
+              view.gst.display = GST_GL_DISPLAY(gst_object_ref(initializeContext.display));
 
             view.wpe.width = initializeContext.width;
             view.wpe.height = initializeContext.height;
@@ -273,25 +286,58 @@ GstEGLImage* WPEThreadedView::image()
     bool dispatchFrameComplete = false;
 
     {
-        GMutexHolder lock(images.mutex);
+        GMutexHolder lock(images_mutex);
 
-        GST_TRACE("pending %" GST_PTR_FORMAT " (%d) committed %" GST_PTR_FORMAT " (%d)", images.pending,
+        GST_TRACE("pending %" GST_PTR_FORMAT " (%d) committed %" GST_PTR_FORMAT " (%d)", egl.pending,
-                  GST_IS_EGL_IMAGE(images.pending) ? GST_MINI_OBJECT_REFCOUNT_VALUE(GST_MINI_OBJECT_CAST(images.pending)) : 0,
+                  GST_IS_EGL_IMAGE(egl.pending) ? GST_MINI_OBJECT_REFCOUNT_VALUE(GST_MINI_OBJECT_CAST(egl.pending)) : 0,
-                  images.committed,
+                  egl.committed,
-                  GST_IS_EGL_IMAGE(images.committed) ? GST_MINI_OBJECT_REFCOUNT_VALUE(GST_MINI_OBJECT_CAST(images.committed)) : 0);
+                  GST_IS_EGL_IMAGE(egl.committed) ? GST_MINI_OBJECT_REFCOUNT_VALUE(GST_MINI_OBJECT_CAST(egl.committed)) : 0);
 
-        if (images.pending) {
+        if (egl.pending) {
-            auto* previousImage = images.committed;
+            auto* previousImage = egl.committed;
-            images.committed = images.pending;
+            egl.committed = egl.pending;
-            images.pending = nullptr;
+            egl.pending = nullptr;
 
             if (previousImage)
                 gst_egl_image_unref(previousImage);
             dispatchFrameComplete = true;
         }
 
-        if (images.committed)
+        if (egl.committed)
-            ret = images.committed;
+            ret = egl.committed;
+    }
+
+    if (dispatchFrameComplete)
+        frameComplete();
+
+    return ret;
+}
+
+GstBuffer* WPEThreadedView::buffer()
+{
+    GstBuffer* ret = nullptr;
+    bool dispatchFrameComplete = false;
+
+    {
+        GMutexHolder lock(images_mutex);
+
+        GST_TRACE("pending %" GST_PTR_FORMAT " (%d) committed %" GST_PTR_FORMAT " (%d)", shm.pending,
+                  GST_IS_BUFFER(shm.pending) ? GST_MINI_OBJECT_REFCOUNT_VALUE(GST_MINI_OBJECT_CAST(shm.pending)) : 0,
+                  shm.committed,
+                  GST_IS_BUFFER(shm.committed) ? GST_MINI_OBJECT_REFCOUNT_VALUE(GST_MINI_OBJECT_CAST(shm.committed)) : 0);
+
+        if (shm.pending) {
+            auto* previousImage = shm.committed;
+            shm.committed = shm.pending;
+            shm.pending = nullptr;
+
+            if (previousImage)
+                gst_buffer_unref(previousImage);
+            dispatchFrameComplete = true;
+        }
+
+        if (shm.committed)
+            ret = shm.committed;
     }
 
     if (dispatchFrameComplete)
@@ -504,13 +550,93 @@ void WPEThreadedView::handleExportedImage(gpointer image)
 
     auto* gstImage = gst_egl_image_new_wrapped(gst.context, eglImage, GST_GL_RGBA, imageContext, s_releaseImage);
     {
-      GMutexHolder lock(images.mutex);
+      GMutexHolder lock(images_mutex);
 
       GST_TRACE("EGLImage %p wrapped in GstEGLImage %" GST_PTR_FORMAT, eglImage, gstImage);
-      images.pending = gstImage;
+      egl.pending = gstImage;
     }
 }
 
+#if ENABLE_SHM_BUFFER_SUPPORT
+struct SHMBufferContext {
+  WPEThreadedView* view;
+  struct wpe_fdo_shm_exported_buffer* buffer;
+};
+
+void WPEThreadedView::releaseSHMBuffer(gpointer data)
+{
+    SHMBufferContext* context = static_cast<SHMBufferContext*>(data);
+    struct ReleaseBufferContext {
+        WPEThreadedView& view;
+        SHMBufferContext* context;
+    } releaseImageContext{ *this, context };
+
+    GSource* source = g_idle_source_new();
+    g_source_set_callback(source,
+        [](gpointer data) -> gboolean {
+            auto& releaseBufferContext = *static_cast<ReleaseBufferContext*>(data);
+            auto& view = releaseBufferContext.view;
+            GMutexHolder lock(view.threading.mutex);
+
+            struct wpe_fdo_shm_exported_buffer* buffer = static_cast<struct wpe_fdo_shm_exported_buffer*>(releaseBufferContext.context->buffer);
+            GST_TRACE("Dispatch release exported buffer %p", buffer);
+            wpe_view_backend_exportable_fdo_dispatch_release_shm_exported_buffer(view.wpe.exportable, buffer);
+            g_cond_signal(&view.threading.cond);
+            return G_SOURCE_REMOVE;
+        },
+        &releaseImageContext, nullptr);
+    g_source_set_priority(source, WPE_GLIB_SOURCE_PRIORITY);
+
+    {
+        GMutexHolder lock(threading.mutex);
+        g_source_attach(source, glib.context);
+        g_cond_wait(&threading.cond, &threading.mutex);
+    }
+
+    g_source_unref(source);
+}
+
+void WPEThreadedView::s_releaseSHMBuffer(gpointer data)
+{
+    SHMBufferContext* context = static_cast<SHMBufferContext*>(data);
+    context->view->releaseSHMBuffer(data);
+    g_slice_free(SHMBufferContext, context);
+}
+
+void WPEThreadedView::handleExportedBuffer(struct wpe_fdo_shm_exported_buffer* buffer)
+{
+    struct wl_shm_buffer* shmBuffer = wpe_fdo_shm_exported_buffer_get_shm_buffer(buffer);
+    auto format = wl_shm_buffer_get_format(shmBuffer);
+    if (format != WL_SHM_FORMAT_ARGB8888 && format != WL_SHM_FORMAT_XRGB8888) {
+        GST_ERROR("Unsupported pixel format: %d", format);
+        return;
+    }
+
+    int32_t width = wl_shm_buffer_get_width(shmBuffer);
+    int32_t height = wl_shm_buffer_get_height(shmBuffer);
+    gint stride = wl_shm_buffer_get_stride(shmBuffer);
+    gsize size = width * height * 4;
+    auto* data = static_cast<uint8_t*>(wl_shm_buffer_get_data(shmBuffer));
+
+    SHMBufferContext* bufferContext = g_slice_new(SHMBufferContext);
+    bufferContext->view = this;
+    bufferContext->buffer = buffer;
+
+    auto* gstBuffer = gst_buffer_new_wrapped_full(GST_MEMORY_FLAG_READONLY, data, size, 0, size, bufferContext, s_releaseSHMBuffer);
+    gsize offsets[1];
+    gint strides[1];
+    offsets[0] = 0;
+    strides[0] = stride;
+    gst_buffer_add_video_meta_full(gstBuffer, GST_VIDEO_FRAME_FLAG_NONE, GST_VIDEO_FORMAT_BGRA, width, height, 1, offsets, strides);
+
+    {
+        GMutexHolder lock(images_mutex);
+        GST_TRACE("SHM buffer %p wrapped in buffer %" GST_PTR_FORMAT, buffer, gstBuffer);
+        shm.pending = gstBuffer;
+    }
+}
+#endif
+
 struct wpe_view_backend_exportable_fdo_egl_client WPEThreadedView::s_exportableClient = {
 #if USE_DEPRECATED_FDO_EGL_IMAGE
     // export_egl_image
@@ -518,7 +644,7 @@ struct wpe_view_backend_exportable_fdo_egl_client WPEThreadedView::s_exportableC
         auto& view = *static_cast<WPEThreadedView*>(data);
         view.handleExportedImage(static_cast<gpointer>(image));
     },
-    nullptr,
+    nullptr, nullptr,
 #else
     // export_egl_image
     nullptr,
@@ -526,9 +652,18 @@ struct wpe_view_backend_exportable_fdo_egl_client WPEThreadedView::s_exportableC
         auto& view = *static_cast<WPEThreadedView*>(data);
         view.handleExportedImage(static_cast<gpointer>(image));
     },
-#endif
+#if ENABLE_SHM_BUFFER_SUPPORT
+    // export_shm_buffer
+    [](void* data, struct wpe_fdo_shm_exported_buffer* buffer) {
+        auto& view = *static_cast<WPEThreadedView*>(data);
+        view.handleExportedBuffer(buffer);
+    },
+#else
+    nullptr,
+#endif // ENABLE_SHM_BUFFER_SUPPORT
+#endif // USE_DEPRECATED_FDO_EGL_IMAGE
     // padding
-    nullptr, nullptr, nullptr
+    nullptr, nullptr
 };
 
 void WPEThreadedView::s_releaseImage(GstEGLImage* image, gpointer data)

ext/wpe/WPEThreadedView.h

@@ -31,6 +31,12 @@ typedef struct _GstGLContext GstGLContext;
 typedef struct _GstGLDisplay GstGLDisplay;
 typedef struct _GstEGLImage GstEGLImage;
 
+#if defined(WPE_FDO_CHECK_VERSION) && WPE_FDO_CHECK_VERSION(1, 5, 0)
+#define ENABLE_SHM_BUFFER_SUPPORT 1
+#else
+#define ENABLE_SHM_BUFFER_SUPPORT 0
+#endif
+
 class WPEThreadedView {
 public:
     WPEThreadedView();
@@ -44,17 +50,25 @@ public:
     void setDrawBackground(gboolean);
 
     GstEGLImage* image();
+    GstBuffer* buffer();
 
     struct wpe_view_backend* backend() const;
 
 protected:
     void handleExportedImage(gpointer);
+#if ENABLE_SHM_BUFFER_SUPPORT
+    void handleExportedBuffer(struct wpe_fdo_shm_exported_buffer*);
+#endif
 
 private:
     void frameComplete();
     void loadUriUnlocked(const gchar*);
 
     void releaseImage(gpointer);
+#if ENABLE_SHM_BUFFER_SUPPORT
+    void releaseSHMBuffer(gpointer);
+    static void s_releaseSHMBuffer(gpointer);
+#endif
 
     static void s_loadEvent(WebKitWebView*, WebKitLoadEvent, gpointer);
 
@@ -90,9 +104,17 @@ private:
         WebKitWebView* view;
     } webkit = { nullptr, nullptr };
 
+    // This mutex guards access to either egl or shm resources declared below,
+    // depending on the runtime behavior.
+    GMutex images_mutex;
+
     struct {
-        GMutex mutex;
+        GstEGLImage* pending;
-        GstEGLImage* pending { nullptr };
+        GstEGLImage* committed;
-        GstEGLImage* committed { nullptr };
+    } egl { nullptr, nullptr };
-    } images;
+
+    struct {
+        GstBuffer* pending;
+        GstBuffer* committed;
+    } shm { nullptr, nullptr };
 };

ext/wpe/gstwpesrc.cpp

@@ -24,7 +24,13 @@
  * The wpesrc element is used to produce a video texture representing a web page
  * rendered off-screen by WPE.
  *
- * ## Example launch line
+ * Starting from WPEBackend-FDO 1.6.x, software rendering support is available. This
+ * features allows wpesrc to be used on machines without GPU, and/or for testing
+ * purpose. To enable it, set the `LIBGL_ALWAYS_SOFTWARE=true` environment
+ * variable and make sure `video/x-raw, format=BGRA` caps are negotiated by the
+ * wpesrc element.
+ *
+ * ## Example launch lines
  *
  * |[
  * gst-launch-1.0 -v wpesrc location="https://gstreamer.freedesktop.org" ! queue ! glimagesink
@@ -32,6 +38,11 @@
  * Shows the GStreamer website homepage
  *
  * |[
+ * LIBGL_ALWAYS_SOFTWARE=true gst-launch-1.0 -v wpesrc num-buffers=50 location="https://gstreamer.freedesktop.org" ! videoconvert ! pngenc ! multifilesink location=/tmp/snapshot-%05d.png
+ * ]|
+ * Saves the first 50 video frames generated for the GStreamer website as PNG files in /tmp.
+ *
+ * |[
  * gst-play-1.0 --videosink gtkglsink wpe://https://gstreamer.freedesktop.org
  * ]|
  * Shows the GStreamer website homepage as played with GstPlayer in a GTK+ window.
@@ -102,6 +113,7 @@ struct _GstWpeSrc
   gboolean draw_background;
 
   GBytes *bytes;
+  gboolean gl_enabled;
 };
 
 static void gst_wpe_src_uri_handler_init (gpointer iface, gpointer data);
@@ -118,9 +130,41 @@ static GstStaticPadTemplate src_factory = GST_STATIC_PAD_TEMPLATE ("src",
         "width = " GST_VIDEO_SIZE_RANGE ", "
         "height = " GST_VIDEO_SIZE_RANGE ", "
         "framerate = " GST_VIDEO_FPS_RANGE ", "
-        "pixel-aspect-ratio = (fraction)1/1," "texture-target = (string)2D")
+        "pixel-aspect-ratio = (fraction)1/1, texture-target = (string)2D;"
+#if ENABLE_SHM_BUFFER_SUPPORT
+        "video/x-raw, "
+        "format = (string) BGRA, "
+        "width = " GST_VIDEO_SIZE_RANGE ", "
+        "height = " GST_VIDEO_SIZE_RANGE ", "
+        "framerate = " GST_VIDEO_FPS_RANGE ", "
+        "pixel-aspect-ratio = (fraction)1/1"
+#endif
+        )
     );
 
+static GstFlowReturn
+gst_wpe_src_create (GstBaseSrc * bsrc, guint64 offset, guint length, GstBuffer ** buf)
+{
+  GstWpeSrc *src = GST_WPE_SRC (bsrc);
+  GstFlowReturn ret = GST_FLOW_ERROR;
+  GstBuffer *locked_buffer;
+
+  GST_OBJECT_LOCK (src);
+  if (src->gl_enabled) {
+    GST_OBJECT_UNLOCK (src);
+    return GST_CALL_PARENT_WITH_DEFAULT (GST_BASE_SRC_CLASS, create, (bsrc, offset, length, buf), ret);
+  }
+
+  locked_buffer = src->view->buffer ();
+
+  if (locked_buffer != NULL) {
+    *buf = gst_buffer_ref (locked_buffer);
+    ret = GST_FLOW_OK;
+  }
+  GST_OBJECT_UNLOCK (src);
+  return ret;
+}
+
 static gboolean
 gst_wpe_src_fill_memory (GstGLBaseSrc * bsrc, GstGLMemory * memory)
 {
@@ -160,13 +204,28 @@ gst_wpe_src_gl_start (GstGLBaseSrc * base_src)
 {
   GstWpeSrc *src = GST_WPE_SRC (base_src);
   gboolean result = TRUE;
+  GstCapsFeatures *caps_features;
+  GstGLContext *context = NULL;
+  GstGLDisplay *display = NULL;
 
   GST_INFO_OBJECT (src, "Starting up");
   GST_OBJECT_LOCK (src);
+
+  caps_features = gst_caps_get_features (base_src->out_caps, 0);
+  if (caps_features != NULL && gst_caps_features_contains (caps_features, GST_CAPS_FEATURE_MEMORY_GL_MEMORY)) {
+    src->gl_enabled = TRUE;
+    context = base_src->context;
+    display = base_src->display;
+  } else {
+    src->gl_enabled = FALSE;
+  }
+
+  GST_DEBUG_OBJECT (src, "Will fill GLMemories: %d\n", src->gl_enabled);
+
   src->view = new WPEThreadedView;
-  result = src->view->initialize (src, base_src->context, base_src->display,
+  result = src->view->initialize (src, context, display,
-      GST_VIDEO_INFO_WIDTH (&base_src->out_info),
+    GST_VIDEO_INFO_WIDTH (&base_src->out_info),
-      GST_VIDEO_INFO_HEIGHT (&base_src->out_info));
+    GST_VIDEO_INFO_HEIGHT (&base_src->out_info));
 
   if (src->bytes != NULL) {
     src->view->loadData (src->bytes);
@@ -484,6 +543,7 @@ gst_wpe_src_class_init (GstWpeSrcClass * klass)
   gst_element_class_add_static_pad_template (gstelement_class, &src_factory);
 
   base_src_class->fixate = GST_DEBUG_FUNCPTR (gst_wpe_src_fixate);
+  base_src_class->create = GST_DEBUG_FUNCPTR (gst_wpe_src_create);
 
   gl_base_src_class->supported_gl_api =
       static_cast < GstGLAPI >

ext/wpe/meson.build

@@ -10,6 +10,7 @@ wpe_dep = dependency('wpe-webkit-1.0', version : '>= 2.24', required : get_optio
 wpe_fdo_dep = dependency('wpebackend-fdo-1.0', required : get_option('wpe'))
 egl_dep = dependency('egl', required : get_option('wpe'))
 xkbcommon_dep = dependency('xkbcommon', version : '>= 0.8', required : get_option('wpe'))
+wl_server_dep = dependency('wayland-server', required : get_option('wpe'))
 
 if not wpe_dep.found() or not wpe_fdo_dep.found() or not egl_dep.found() or not xkbcommon_dep.found()
   subdir_done()
@@ -17,7 +18,7 @@ endif
 
 gstwpe = library('gstwpe',
   ['WPEThreadedView.cpp', 'gstwpesrc.cpp'],
-  dependencies : [egl_dep, wpe_dep, wpe_fdo_dep, gstvideo_dep, gstbase_dep, gstgl_dep, xkbcommon_dep],
+  dependencies : [egl_dep, wpe_dep, wpe_fdo_dep, gstvideo_dep, gstbase_dep, gstgl_dep, xkbcommon_dep, wl_server_dep],
   cpp_args : gst_plugins_bad_args + ['-DHAVE_CONFIG_H=1'],
   include_directories : [configinc],
   install : true,