diff --git a/gst-libs/gst/video/video-frame.c b/gst-libs/gst/video/video-frame.c
index 537cf70923..01f23c086e 100644
--- a/gst-libs/gst/video/video-frame.c
+++ b/gst-libs/gst/video/video-frame.c
@@ -106,6 +106,9 @@ gst_video_frame_map_id (GstVideoFrame * frame, GstVideoInfo * info,
     }
   }
   frame->buffer = gst_buffer_ref (buffer);
+  if ((flags & GST_VIDEO_FRAME_MAP_FLAG_NO_REF) == 0)
+    gst_buffer_ref (frame->buffer);
+
   frame->meta = meta;
 
   /* buffer flags enhance the frame flags */
@@ -189,11 +192,13 @@ gst_video_frame_unmap (GstVideoFrame * frame)
   GstBuffer *buffer;
   GstVideoMeta *meta;
   gint i;
+  GstMapFlags flags;
 
   g_return_if_fail (frame != NULL);
 
   buffer = frame->buffer;
   meta = frame->meta;
+  flags = frame->map[0].flags;
 
   if (meta) {
     for (i = 0; i < frame->info.finfo->n_planes; i++) {
@@ -202,7 +207,9 @@ gst_video_frame_unmap (GstVideoFrame * frame)
   } else {
     gst_buffer_unmap (buffer, &frame->map[0]);
   }
-  gst_buffer_unref (buffer);
+
+  if ((flags & GST_VIDEO_FRAME_MAP_FLAG_NO_REF) == 0)
+    gst_buffer_unref (frame->buffer);
 }
 
 /**
diff --git a/gst-libs/gst/video/video-frame.h b/gst-libs/gst/video/video-frame.h
index 627fab0a2d..f8e6304635 100644
--- a/gst-libs/gst/video/video-frame.h
+++ b/gst-libs/gst/video/video-frame.h
@@ -149,6 +149,24 @@ typedef enum {
   GST_VIDEO_BUFFER_FLAG_LAST        = (GST_BUFFER_FLAG_LAST << 8)
 } GstVideoBufferFlags;
 
+/**
+ * GstVideoBufferFlags:
+ * @GST_VIDEO_FRAME_MAP_FLAG_NO_REF:  Don't take another reference of the buffer and store it in
+ *                                    the GstVideoFrame. This makes sure that the buffer stays
+ *                                    writable while the frame is mapped, but requires that the
+ *                                    buffer reference stays valid until the frame is unmapped again.
+ * @GST_VIDEO_FRAME_MAP_FLAG_LAST:    Offset to define more flags
+ *
+ * Additional mapping flags for gst_video_frame_map().
+ *
+ * Since: 1.6
+ */
+typedef enum {
+  GST_VIDEO_FRAME_MAP_FLAG_NO_REF   = (GST_MAP_FLAG_LAST << 0),
+  GST_VIDEO_FRAME_MAP_FLAG_LAST     = (GST_MAP_FLAG_LAST << 8)
+  /* 8 more flags possible afterwards */
+} GstVideoFrameMapFlags;
+
 G_END_DECLS
 
 #endif /* __GST_VIDEO_FRAME_H__ */