diff --git a/sys/msdk/gstmsdkdec.c b/sys/msdk/gstmsdkdec.c
index 7e456a4923..cb4e15ea0f 100644
--- a/sys/msdk/gstmsdkdec.c
+++ b/sys/msdk/gstmsdkdec.c
@@ -681,19 +681,19 @@ gst_msdkdec_start (GstVideoDecoder * decoder)
 {
   GstMsdkDec *thiz = GST_MSDKDEC (decoder);
 
+  /* TODO: Currently d3d allocator is not implemented.
+   * So decoder uses system memory by default on Windows.
+   */
+#ifndef _WIN32
+  thiz->use_video_memory = TRUE;
+#else
+  thiz->use_video_memory = FALSE;
+#endif
+
   if (gst_msdk_context_prepare (GST_ELEMENT_CAST (thiz), &thiz->context)) {
     GST_INFO_OBJECT (thiz, "Found context %" GST_PTR_FORMAT " from neighbour",
         thiz->context);
 
-    /* TODO: Currently d3d allocator is not implemented.
-     * So decoder uses system memory by default on Windows.
-     */
-#ifndef _WIN32
-    thiz->use_video_memory = TRUE;
-#else
-    thiz->use_video_memory = FALSE;
-#endif
-
     if (gst_msdk_context_get_job_type (thiz->context) & GST_MSDK_JOB_DECODER) {
       GstMsdkContext *parent_context, *msdk_context;