diff --git a/sys/msdk/Makefile.am b/sys/msdk/Makefile.am
index 104ee1e5b2..d968c7602b 100644
--- a/sys/msdk/Makefile.am
+++ b/sys/msdk/Makefile.am
@@ -54,7 +54,8 @@ libgstmsdk_la_LIBADD = \
 if USE_MSDK_LIBVA
 libgstmsdk_la_SOURCES += \
 	msdk.c \
-	msdk_libva.c
+	msdk_libva.c \
+	gstmsdkallocator_libva.c
 libgstmsdk_la_CFLAGS += \
 	$(LIBVA_DRM_CFLAGS)
 libgstmsdk_la_LIBADD += \
diff --git a/sys/msdk/gstmsdkallocator.h b/sys/msdk/gstmsdkallocator.h
new file mode 100644
index 0000000000..64cd6a1644
--- /dev/null
+++ b/sys/msdk/gstmsdkallocator.h
@@ -0,0 +1,62 @@
+/* GStreamer Intel MSDK plugin
+ * Copyright (c) 2018, Intel Corporation
+ * Copyright (c) 2018, Igalia S.L.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ * 3. Neither the name of the copyright holder nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGDECE
+ * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
+ * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef GST_MSDK_ALLOCATOR_H_
+#define GST_MSDK_ALLOCATOR_H_
+
+#include "msdk.h"
+#include "gstmsdkcontext.h"
+
+G_BEGIN_DECLS
+
+typedef struct _GstMsdkMemoryID GstMsdkMemoryID;
+
+struct _GstMsdkMemoryID {
+  mfxU32 fourcc;
+
+#ifndef _WIN32
+  VASurfaceID *surface;
+  VAImage image;
+#endif
+};
+
+mfxStatus gst_msdk_frame_alloc(mfxHDL pthis, mfxFrameAllocRequest *req, mfxFrameAllocResponse *resp);
+mfxStatus gst_msdk_frame_free(mfxHDL pthis, mfxFrameAllocResponse *resp);
+mfxStatus gst_msdk_frame_lock(mfxHDL pthis, mfxMemId mid, mfxFrameData *ptr);
+mfxStatus gst_msdk_frame_unlock(mfxHDL pthis, mfxMemId mid, mfxFrameData *ptr);
+mfxStatus gst_msdk_frame_get_hdl(mfxHDL pthis, mfxMemId mid, mfxHDL *hdl);
+
+void gst_msdk_set_frame_allocator (GstMsdkContext * context);
+
+G_END_DECLS
+
+#endif /* GST_MSDK_ALLOCATOR_H_ */
diff --git a/sys/msdk/gstmsdkallocator_libva.c b/sys/msdk/gstmsdkallocator_libva.c
new file mode 100644
index 0000000000..364ee4945e
--- /dev/null
+++ b/sys/msdk/gstmsdkallocator_libva.c
@@ -0,0 +1,319 @@
+/* GStreamer Intel MSDK plugin
+ * Copyright (c) 2018, Intel Corporation
+ * Copyright (c) 2018, Igalia S.L.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice,
+ *    this list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ * 3. Neither the name of the copyright holder nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGDECE
+ * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
+ * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <va/va.h>
+#include "gstmsdkallocator.h"
+#include "msdk_libva.h"
+
+mfxStatus
+gst_msdk_frame_alloc (mfxHDL pthis, mfxFrameAllocRequest * req,
+    mfxFrameAllocResponse * resp)
+{
+  VAStatus va_status;
+  mfxStatus status = MFX_ERR_NONE;
+  gint i;
+  guint format;
+  guint va_fourcc = 0;
+  VASurfaceID *surfaces = NULL;
+  VASurfaceAttrib attrib;
+  mfxMemId *mids = NULL;
+  GstMsdkContext *context = (GstMsdkContext *) pthis;
+  GstMsdkMemoryID *msdk_mids = NULL;
+  GstMsdkAllocResponse *msdk_resp = NULL;
+  mfxU32 fourcc = req->Info.FourCC;
+  mfxU16 surfaces_num = req->NumFrameSuggested;
+
+  if (req->Type & MFX_MEMTYPE_EXTERNAL_FRAME) {
+    GstMsdkAllocResponse *cached =
+        gst_msdk_context_get_cached_alloc_responses_by_request (context, req);
+    if (cached) {
+      *resp = *cached->response;
+      return MFX_ERR_NONE;
+    }
+  }
+
+  /* The VA API does not define any surface types and the application can use either
+   * MFX_MEMTYPE_VIDEO_MEMORY_DECODER_TARGET or
+   * MFX_MEMTYPE_VIDEO_MEMORY_PROCESSOR_TARGET to indicate data in video memory.
+   */
+  if (!(req->Type & (MFX_MEMTYPE_VIDEO_MEMORY_DECODER_TARGET |
+              MFX_MEMTYPE_VIDEO_MEMORY_PROCESSOR_TARGET)))
+    return MFX_ERR_UNSUPPORTED;
+
+  va_fourcc = gst_msdk_get_va_fourcc_from_mfx_fourcc (fourcc);
+
+  msdk_mids =
+      (GstMsdkMemoryID *) g_slice_alloc0 (surfaces_num *
+      sizeof (GstMsdkMemoryID));
+  mids = (mfxMemId *) g_slice_alloc0 (surfaces_num * sizeof (mfxMemId));
+  surfaces =
+      (VASurfaceID *) g_slice_alloc0 (surfaces_num * sizeof (VASurfaceID));
+  msdk_resp =
+      (GstMsdkAllocResponse *) g_slice_alloc0 (sizeof (GstMsdkAllocResponse));
+
+  if (va_fourcc != VA_FOURCC_P208) {
+    attrib.type = VASurfaceAttribPixelFormat;
+    attrib.flags = VA_SURFACE_ATTRIB_SETTABLE;
+    attrib.value.type = VAGenericValueTypeInteger;
+    attrib.value.value.i = va_fourcc;
+
+    format =
+        gst_msdk_get_va_rt_format_from_mfx_rt_format (req->Info.ChromaFormat);
+
+    va_status = vaCreateSurfaces (gst_msdk_context_get_handle (context),
+        format,
+        req->Info.Width, req->Info.Height, surfaces, surfaces_num, &attrib, 1);
+
+    status = gst_msdk_get_mfx_status_from_va_status (va_status);
+    if (status != MFX_ERR_NONE) {
+      GST_WARNING ("failed to create VA surface");
+      return status;
+    }
+
+    for (i = 0; i < surfaces_num; i++) {
+      msdk_mids[i].surface = &surfaces[i];
+      mids[i] = (mfxMemId *) & msdk_mids[i];
+    }
+  } else {
+    /* This is requested from the driver when h265 encoding.
+     * These buffers will be used inside the driver and released by
+     * gst_msdk_frame_free functions. Application doesn't need to handle these buffers.
+     *
+     * See https://github.com/Intel-Media-SDK/samples/issues/13 for more details.
+     */
+    VAContextID context_id = req->AllocId;
+    gint width32 = 32 * ((req->Info.Width + 31) >> 5);
+    gint height32 = 32 * ((req->Info.Height + 31) >> 5);
+    guint64 codedbuf_size = (width32 * height32) * 400LL / (16 * 16);
+
+    for (i = 0; i < surfaces_num; i++) {
+      VABufferID coded_buf;
+
+      va_status = vaCreateBuffer (gst_msdk_context_get_handle (context),
+          context_id, VAEncCodedBufferType, codedbuf_size, 1, NULL, &coded_buf);
+
+      status = gst_msdk_get_mfx_status_from_va_status (va_status);
+      if (status < MFX_ERR_NONE) {
+        GST_ERROR ("failed to create buffer");
+        return status;
+      }
+
+      surfaces[i] = coded_buf;
+      msdk_mids[i].surface = &surfaces[i];
+      msdk_mids[i].fourcc = fourcc;
+      mids[i] = (mfxMemId *) & msdk_mids[i];
+    }
+  }
+
+  resp->mids = mids;
+  resp->NumFrameActual = surfaces_num;
+
+  msdk_resp->response = resp;
+  msdk_resp->mem_ids = mids;
+  msdk_resp->request = *req;
+
+  gst_msdk_context_add_alloc_response (context, msdk_resp);
+
+  return status;
+}
+
+mfxStatus
+gst_msdk_frame_free (mfxHDL pthis, mfxFrameAllocResponse * resp)
+{
+  GstMsdkContext *context = (GstMsdkContext *) pthis;
+  VAStatus va_status = VA_STATUS_SUCCESS;
+  mfxStatus status;
+  GstMsdkMemoryID *mem_id;
+  VADisplay dpy;
+  gint i;
+
+  if (!gst_msdk_context_remove_alloc_response (context, resp))
+    return MFX_ERR_NONE;
+
+  mem_id = resp->mids[0];
+  dpy = gst_msdk_context_get_handle (context);
+
+  if (mem_id->fourcc != MFX_FOURCC_P8) {
+    /* Make sure that all the vaImages are destroyed */
+    for (i = 0; i < resp->NumFrameActual; i++) {
+      GstMsdkMemoryID *mem = resp->mids[i];
+      vaDestroyImage (dpy, mem->image.image_id);
+    }
+
+    va_status =
+        vaDestroySurfaces (dpy, (VASurfaceID *) mem_id->surface,
+        resp->NumFrameActual);
+  } else {
+    VASurfaceID *surfaces = mem_id->surface;
+
+    for (i = 0; i < resp->NumFrameActual; i++) {
+      va_status = vaDestroyBuffer (dpy, surfaces[i]);
+    }
+  }
+
+  g_slice_free1 (resp->NumFrameActual * sizeof (VASurfaceID), mem_id->surface);
+  g_slice_free1 (resp->NumFrameActual * sizeof (GstMsdkMemoryID), mem_id);
+  g_slice_free1 (resp->NumFrameActual * sizeof (mfxMemId), resp->mids);
+
+  status = gst_msdk_get_mfx_status_from_va_status (va_status);
+  return status;
+}
+
+mfxStatus
+gst_msdk_frame_lock (mfxHDL pthis, mfxMemId mid, mfxFrameData * data)
+{
+  GstMsdkContext *context = (GstMsdkContext *) pthis;
+  VAStatus va_status;
+  mfxStatus status;
+  mfxU8 *buf = NULL;
+  VASurfaceID *va_surface;
+  VADisplay dpy;
+  GstMsdkMemoryID *mem_id;
+
+  mem_id = (GstMsdkMemoryID *) mid;
+  va_surface = mem_id->surface;
+  dpy = gst_msdk_context_get_handle (context);
+
+  if (mem_id->fourcc != MFX_FOURCC_P8) {
+    va_status = vaDeriveImage (dpy, *va_surface, &mem_id->image);
+    status = gst_msdk_get_mfx_status_from_va_status (va_status);
+
+    if (status != MFX_ERR_NONE) {
+      GST_WARNING ("failed to derive image");
+      return status;
+    }
+
+    va_status = vaMapBuffer (dpy, mem_id->image.buf, (void **) &buf);
+    status = gst_msdk_get_mfx_status_from_va_status (va_status);
+
+    if (status != MFX_ERR_NONE) {
+      GST_WARNING ("failed to map");
+      return status;
+    }
+
+    switch (mem_id->image.format.fourcc) {
+      case VA_FOURCC_NV12:
+        data->Pitch = mem_id->image.pitches[0];
+        data->Y = buf + mem_id->image.offsets[0];
+        data->UV = buf + mem_id->image.offsets[1];
+        break;
+      case VA_FOURCC_YV12:
+        data->Pitch = mem_id->image.pitches[0];
+        data->Y = buf + mem_id->image.offsets[0];
+        data->U = buf + mem_id->image.offsets[2];
+        data->V = buf + mem_id->image.offsets[1];
+        break;
+      case VA_FOURCC_YUY2:
+        data->Pitch = mem_id->image.pitches[0];
+        data->Y = buf + mem_id->image.offsets[0];
+        data->U = data->Y + 1;
+        data->V = data->Y + 3;
+        break;
+      case VA_FOURCC_UYVY:
+        data->Pitch = mem_id->image.pitches[0];
+        data->Y = buf + mem_id->image.offsets[0];
+        data->U = data->U + 1;
+        data->V = data->U + 2;
+        break;
+      case VA_FOURCC_ARGB:
+        data->Pitch = mem_id->image.pitches[0];
+        data->R = buf + mem_id->image.offsets[0];
+        data->G = data->R + 1;
+        data->B = data->R + 2;
+        data->A = data->R + 3;
+        break;
+    }
+  } else {
+    VACodedBufferSegment *coded_buffer_segment;
+    va_status =
+        vaMapBuffer (dpy, *va_surface, (void **) (&coded_buffer_segment));
+    status = gst_msdk_get_mfx_status_from_va_status (va_status);
+    if (MFX_ERR_NONE == status)
+      data->Y = (mfxU8 *) coded_buffer_segment->buf;
+  }
+
+  return status;
+}
+
+mfxStatus
+gst_msdk_frame_unlock (mfxHDL pthis, mfxMemId mid, mfxFrameData * ptr)
+{
+  GstMsdkContext *context = (GstMsdkContext *) pthis;
+  VAStatus va_status;
+  mfxStatus status;
+  VADisplay dpy;
+  GstMsdkMemoryID *mem_id;
+
+  mem_id = (GstMsdkMemoryID *) mid;
+  dpy = gst_msdk_context_get_handle (context);
+
+  if (mem_id->fourcc != MFX_FOURCC_P8) {
+    vaUnmapBuffer (dpy, mem_id->image.buf);
+    va_status = vaDestroyImage (dpy, mem_id->image.image_id);
+  } else {
+    va_status = vaUnmapBuffer (dpy, *(mem_id->surface));
+  }
+
+  status = gst_msdk_get_mfx_status_from_va_status (va_status);
+
+  return status;
+}
+
+mfxStatus
+gst_msdk_frame_get_hdl (mfxHDL pthis, mfxMemId mid, mfxHDL * hdl)
+{
+  GstMsdkMemoryID *mem_id;
+
+  if (!hdl || !mid)
+    return MFX_ERR_INVALID_HANDLE;
+
+  mem_id = mid;
+  *hdl = mem_id->surface;
+
+  return MFX_ERR_NONE;
+}
+
+void
+gst_msdk_set_frame_allocator (GstMsdkContext * context)
+{
+  mfxFrameAllocator gst_msdk_frame_allocator = {
+    .pthis = context,
+    .Alloc = gst_msdk_frame_alloc,
+    .Lock = gst_msdk_frame_lock,
+    .Unlock = gst_msdk_frame_unlock,
+    .GetHDL = gst_msdk_frame_get_hdl,
+    .Free = gst_msdk_frame_free,
+  };
+
+  MFXVideoCORE_SetFrameAllocator (gst_msdk_context_get_session (context),
+      &gst_msdk_frame_allocator);
+}
diff --git a/sys/msdk/gstmsdkcontext.c b/sys/msdk/gstmsdkcontext.c
index 851d133a58..176c4d5246 100644
--- a/sys/msdk/gstmsdkcontext.c
+++ b/sys/msdk/gstmsdkcontext.c
@@ -211,3 +211,78 @@ gst_msdk_context_get_fd (GstMsdkContext * context)
   return -1;
 #endif
 }
+
+static gint
+_find_response (gconstpointer resp, gconstpointer comp_resp)
+{
+  GstMsdkAllocResponse *cached_resp = (GstMsdkAllocResponse *) resp;
+  mfxFrameAllocResponse *_resp = (mfxFrameAllocResponse *) comp_resp;
+
+  return cached_resp ? cached_resp->mem_ids != _resp->mids : -1;
+}
+
+static gint
+_find_request (gconstpointer resp, gconstpointer req)
+{
+  GstMsdkAllocResponse *cached_resp = (GstMsdkAllocResponse *) resp;
+  mfxFrameAllocRequest *_req = (mfxFrameAllocRequest *) req;
+
+  return cached_resp ? cached_resp->request.Type != _req->Type : -1;
+}
+
+GstMsdkAllocResponse *
+gst_msdk_context_get_cached_alloc_responses (GstMsdkContext * context,
+    mfxFrameAllocResponse * resp)
+{
+  GstMsdkContextPrivate *priv = context->priv;
+  GList *l =
+      g_list_find_custom (priv->cached_alloc_responses, resp, _find_response);
+
+  if (l)
+    return l->data;
+  else
+    return NULL;
+}
+
+GstMsdkAllocResponse *
+gst_msdk_context_get_cached_alloc_responses_by_request (GstMsdkContext *
+    context, mfxFrameAllocRequest * req)
+{
+  GstMsdkContextPrivate *priv = context->priv;
+  GList *l =
+      g_list_find_custom (priv->cached_alloc_responses, req, _find_request);
+
+  if (l)
+    return l->data;
+  else
+    return NULL;
+}
+
+void
+gst_msdk_context_add_alloc_response (GstMsdkContext * context,
+    GstMsdkAllocResponse * resp)
+{
+  context->priv->cached_alloc_responses =
+      g_list_append (context->priv->cached_alloc_responses, resp);
+}
+
+gboolean
+gst_msdk_context_remove_alloc_response (GstMsdkContext * context,
+    mfxFrameAllocResponse * resp)
+{
+  GstMsdkAllocResponse *msdk_resp;
+  GstMsdkContextPrivate *priv = context->priv;
+  GList *l =
+      g_list_find_custom (priv->cached_alloc_responses, resp, _find_response);
+
+  if (!l)
+    return FALSE;
+
+  msdk_resp = l->data;
+
+  g_slice_free1 (sizeof (GstMsdkAllocResponse), msdk_resp);
+  priv->cached_alloc_responses =
+      g_list_delete_link (priv->cached_alloc_responses, l);
+
+  return TRUE;
+}
diff --git a/sys/msdk/gstmsdkcontext.h b/sys/msdk/gstmsdkcontext.h
index 2f20ff9a55..7aae0b601f 100644
--- a/sys/msdk/gstmsdkcontext.h
+++ b/sys/msdk/gstmsdkcontext.h
@@ -84,6 +84,34 @@ mfxSession gst_msdk_context_get_session (GstMsdkContext * context);
 gpointer gst_msdk_context_get_handle (GstMsdkContext * context);
 gint gst_msdk_context_get_fd (GstMsdkContext * context);
 
+/* GstMsdkContext contains mfxFrameAllocResponses,
+ * if app calls MFXVideoCORE_SetFrameAllocator.
+ */
+typedef struct _GstMsdkAllocResponse GstMsdkAllocResponse;
+
+struct _GstMsdkAllocResponse {
+  mfxFrameAllocResponse *response;
+  mfxFrameAllocRequest request;
+  mfxMemId *mem_ids;
+  gint num_used_memory;
+};
+
+GstMsdkAllocResponse *
+gst_msdk_context_get_cached_alloc_responses (GstMsdkContext * context,
+    mfxFrameAllocResponse * resp);
+
+GstMsdkAllocResponse *
+gst_msdk_context_get_cached_alloc_responses_by_request (GstMsdkContext * context,
+    mfxFrameAllocRequest * req);
+
+void
+gst_msdk_context_add_alloc_response (GstMsdkContext * context,
+    GstMsdkAllocResponse * resp);
+
+gboolean
+gst_msdk_context_remove_alloc_response (GstMsdkContext * context,
+    mfxFrameAllocResponse * resp);
+
 G_END_DECLS
 
 #endif /* GST_MSDK_CONTEXT_H */
diff --git a/sys/msdk/meson.build b/sys/msdk/meson.build
index 041b1fa879..222883901a 100644
--- a/sys/msdk/meson.build
+++ b/sys/msdk/meson.build
@@ -19,7 +19,7 @@ msdk_sources = [
 if host_machine.system() == 'windows'
   msdk_sources += 'msdk_d3d.c'
 else
-  msdk_sources += 'msdk_libva.c'
+  msdk_sources += ['msdk_libva.c', 'gstmsdkallocator_libva.c']
 endif
 
 msdk_root = run_command(python3, '-c', 'import os; print(os.environ.get("INTELMEDIASDKROOT", os.environ.get("MFX_HOME", "")))').stdout().strip()