diff --git a/sys/va/gstvaallocator.c b/sys/va/gstvaallocator.c
index 3207a0715f..b3b7ad422d 100644
--- a/sys/va/gstvaallocator.c
+++ b/sys/va/gstvaallocator.c
@@ -558,6 +558,7 @@ struct _GstVaAllocator
 
   GstVaDisplay *display;
   gboolean use_derived;
+  GArray *surface_formats;
 };
 
 typedef struct _GstVaMemory GstVaMemory;
@@ -588,6 +589,7 @@ gst_va_allocator_dispose (GObject * object)
   GstVaAllocator *self = GST_VA_ALLOCATOR (object);
 
   gst_clear_object (&self->display);
+  g_clear_pointer (&self->surface_formats, g_array_unref);
 
   G_OBJECT_CLASS (gst_va_allocator_parent_class)->dispose (object);
 }
@@ -864,15 +866,20 @@ gst_va_allocator_alloc (GstAllocator * allocator,
 
   self = GST_VA_ALLOCATOR (allocator);
 
-  format = GST_VIDEO_INFO_FORMAT (&params->info);
-  if (gst_va_video_format_is_extra (format))
-    format = GST_VIDEO_FORMAT_NV12;
+  format =
+      gst_va_video_surface_format_from_image_format (GST_VIDEO_INFO_FORMAT
+      (&params->info), self->surface_formats);
+  if (format == GST_VIDEO_FORMAT_UNKNOWN) {
+    GST_ERROR_OBJECT (allocator, "Unsupported format: %s",
+        gst_video_format_to_string (GST_VIDEO_INFO_FORMAT (&params->info)));
+    return NULL;
+  }
 
   fourcc = gst_va_fourcc_from_video_format (format);
   rt_format = gst_va_chroma_from_video_format (format);
   if (fourcc == 0 || rt_format == 0) {
     GST_ERROR_OBJECT (allocator, "Unsupported format: %s",
-        gst_video_format_to_string (format));
+        gst_video_format_to_string (GST_VIDEO_INFO_FORMAT (&params->info)));
     return NULL;
   }
 
@@ -902,7 +909,7 @@ gst_va_allocator_alloc (GstAllocator * allocator,
 }
 
 GstAllocator *
-gst_va_allocator_new (GstVaDisplay * display)
+gst_va_allocator_new (GstVaDisplay * display, GArray * surface_formats)
 {
   GstVaAllocator *self;
 
@@ -910,6 +917,7 @@ gst_va_allocator_new (GstVaDisplay * display)
 
   self = g_object_new (GST_TYPE_VA_ALLOCATOR, NULL);
   self->display = gst_object_ref (display);
+  self->surface_formats = surface_formats;
   gst_object_ref_sink (self);
 
   return GST_ALLOCATOR (self);
diff --git a/sys/va/gstvaallocator.h b/sys/va/gstvaallocator.h
index 2b37bc4cb2..f25b065f11 100644
--- a/sys/va/gstvaallocator.h
+++ b/sys/va/gstvaallocator.h
@@ -50,7 +50,8 @@ G_DECLARE_FINAL_TYPE (GstVaAllocator, gst_va_allocator, GST, VA_ALLOCATOR, GstAl
 
 #define GST_ALLOCATOR_VASURFACE   "VAMemory"
 
-GstAllocator *        gst_va_allocator_new                (GstVaDisplay * display);
+GstAllocator *        gst_va_allocator_new                (GstVaDisplay * display,
+                                                           GArray * surface_formats);
 GstMemory *           gst_va_allocator_alloc              (GstAllocator * allocator,
                                                            GstVaAllocationParams * params);
 gboolean              gst_va_allocator_try                (GstAllocator * allocator,
diff --git a/sys/va/gstvacaps.c b/sys/va/gstvacaps.c
index 0ef7cd8454..92dd4673e9 100644
--- a/sys/va/gstvacaps.c
+++ b/sys/va/gstvacaps.c
@@ -140,25 +140,6 @@ _gst_caps_set_format_array (GstCaps * caps, GArray * formats)
   return TRUE;
 }
 
-/* extra formats to add to raw caps bacause *in theory* all drivers
- * could create images from surface's native format (NV12) to
- * these. */
-static const GstVideoFormat extra_formats[] = {
-  GST_VIDEO_FORMAT_I420, GST_VIDEO_FORMAT_YV12
-};
-
-gboolean
-gst_va_video_format_is_extra (guint format)
-{
-  guint i;
-
-  for (i = 0; i < G_N_ELEMENTS (extra_formats); i++) {
-    if (extra_formats[i] == format)
-      return TRUE;
-  }
-  return FALSE;
-}
-
 GstCaps *
 gst_va_create_raw_caps_from_config (GstVaDisplay * display, VAConfigID config)
 {
diff --git a/sys/va/gstvacaps.h b/sys/va/gstvacaps.h
index daf4416b67..140ede193d 100644
--- a/sys/va/gstvacaps.h
+++ b/sys/va/gstvacaps.h
@@ -37,7 +37,5 @@ VASurfaceAttrib *     gst_va_get_surface_attribs          (GstVaDisplay * displa
 GstCaps *             gst_va_create_raw_caps_from_config  (GstVaDisplay * display,
                                                            VAConfigID config);
 
-gboolean              gst_va_video_format_is_extra        (guint format);
-
 G_END_DECLS
 
diff --git a/sys/va/gstvadecoder.c b/sys/va/gstvadecoder.c
index 2c754849da..36a6a90927 100644
--- a/sys/va/gstvadecoder.c
+++ b/sys/va/gstvadecoder.c
@@ -26,6 +26,7 @@
 
 #include "gstvacaps.h"
 #include "gstvadisplay_wrapped.h"
+#include "gstvavideoformat.h"
 
 struct _GstVaDecoder
 {
@@ -447,6 +448,50 @@ gst_va_decoder_get_mem_types (GstVaDecoder * self)
   return ret;
 }
 
+GArray *
+gst_va_decoder_get_surface_formats (GstVaDecoder * self)
+{
+  GArray *formats;
+  GstVideoFormat format;
+  VASurfaceAttrib *attribs;
+  guint i, attrib_count;
+
+  g_return_val_if_fail (GST_IS_VA_DECODER (self), NULL);
+
+  if (!gst_va_decoder_is_open (self))
+    return NULL;
+
+  attribs = gst_va_get_surface_attribs (self->display, self->config,
+      &attrib_count);
+  if (!attribs)
+    return NULL;
+
+  formats = g_array_new (FALSE, FALSE, sizeof (GstVideoFormat));
+
+  for (i = 0; i < attrib_count; i++) {
+    if (attribs[i].value.type != VAGenericValueTypeInteger)
+      continue;
+    switch (attribs[i].type) {
+      case VASurfaceAttribPixelFormat:
+        format = gst_va_video_format_from_va_fourcc (attribs[i].value.value.i);
+        if (format != GST_VIDEO_FORMAT_UNKNOWN)
+          g_array_append_val (formats, format);
+        break;
+      default:
+        break;
+    }
+  }
+
+  g_free (attribs);
+
+  if (formats->len == 0) {
+    g_array_unref (formats);
+    return NULL;
+  }
+
+  return formats;
+}
+
 gboolean
 gst_va_decoder_add_param_buffer (GstVaDecoder * self, GstVaDecodePicture * pic,
     gint type, gpointer data, gsize size)
diff --git a/sys/va/gstvadecoder.h b/sys/va/gstvadecoder.h
index 4d685ee868..37db1224d1 100644
--- a/sys/va/gstvadecoder.h
+++ b/sys/va/gstvadecoder.h
@@ -51,6 +51,7 @@ GstCaps *             gst_va_decoder_get_sinkpad_caps     (GstVaDecoder * self);
 gboolean              gst_va_decoder_has_profile          (GstVaDecoder * self,
 							   VAProfile profile);
 gint                  gst_va_decoder_get_mem_types        (GstVaDecoder * self);
+GArray *              gst_va_decoder_get_surface_formats  (GstVaDecoder * self);
 
 gboolean              gst_va_decoder_add_param_buffer     (GstVaDecoder * self,
 							   GstVaDecodePicture * pic,
diff --git a/sys/va/gstvah264dec.c b/sys/va/gstvah264dec.c
index 5d470ae7b6..d2db9346eb 100644
--- a/sys/va/gstvah264dec.c
+++ b/sys/va/gstvah264dec.c
@@ -1104,8 +1104,11 @@ _create_allocator (GstVaH264Dec * self, GstCaps * caps, guint * size)
 
   if (_caps_is_dmabuf (self, caps))
     allocator = gst_va_dmabuf_allocator_new (display);
-  else
-    allocator = gst_va_allocator_new (display);
+  else {
+    GArray *surface_formats =
+        gst_va_decoder_get_surface_formats (self->decoder);
+    allocator = gst_va_allocator_new (display, surface_formats);
+  }
 
   gst_object_unref (display);
 
diff --git a/sys/va/gstvavideoformat.c b/sys/va/gstvavideoformat.c
index fbf8c53e90..7e42bca13c 100644
--- a/sys/va/gstvavideoformat.c
+++ b/sys/va/gstvavideoformat.c
@@ -204,3 +204,35 @@ gst_va_video_format_from_va_image_format (const VAImageFormat * va_format)
 
   return map ? map->format : GST_VIDEO_FORMAT_UNKNOWN;
 }
+
+GstVideoFormat
+gst_va_video_surface_format_from_image_format (GstVideoFormat image_format,
+    GArray * surface_formats)
+{
+  GstVideoFormat surface_format;
+  guint i, image_chroma, surface_chroma;
+
+  if (image_format == GST_VIDEO_FORMAT_UNKNOWN)
+    return GST_VIDEO_FORMAT_UNKNOWN;
+
+  if (!surface_formats || surface_formats->len == 0)
+    return GST_VIDEO_FORMAT_UNKNOWN;
+
+  image_chroma = gst_va_chroma_from_video_format (image_format);
+  if (image_chroma == 0)
+    return GST_VIDEO_FORMAT_UNKNOWN;
+
+  for (i = 0; i < surface_formats->len; i++) {
+    surface_format = g_array_index (surface_formats, GstVideoFormat, i);
+
+    if (surface_format == image_format)
+      return surface_format;
+
+    surface_chroma = gst_va_chroma_from_video_format (surface_format);
+
+    if (surface_chroma == image_chroma)
+      return surface_format;
+  }
+
+  return GST_VIDEO_FORMAT_UNKNOWN;
+}
diff --git a/sys/va/gstvavideoformat.h b/sys/va/gstvavideoformat.h
index 1856c172d0..e6c25b2e7b 100644
--- a/sys/va/gstvavideoformat.h
+++ b/sys/va/gstvavideoformat.h
@@ -30,5 +30,7 @@ guint                 gst_va_fourcc_from_video_format     (GstVideoFormat format
 guint                 gst_va_chroma_from_video_format     (GstVideoFormat format);
 const VAImageFormat * gst_va_image_format_from_video_format (GstVideoFormat format);
 GstVideoFormat        gst_va_video_format_from_va_image_format (const VAImageFormat * va_format);
+GstVideoFormat        gst_va_video_surface_format_from_image_format (GstVideoFormat image_format,
+								     GArray * surface_formats);
 
 G_END_DECLS