From f23d4475b3d9aa6f16a5d37f8dd01ace4db66177 Mon Sep 17 00:00:00 2001
From: Seungha Yang <seungha@centricular.com>
Date: Mon, 27 Nov 2023 01:11:04 +0900
Subject: [PATCH] d3d11convert: Allow MSAA only for native DXGI formats

Enable MSAA only if converter can render directly on texture
without post-processing

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/5717>
---
 .../gst-plugins-bad/sys/d3d11/gstd3d11convert.cpp     | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/subprojects/gst-plugins-bad/sys/d3d11/gstd3d11convert.cpp b/subprojects/gst-plugins-bad/sys/d3d11/gstd3d11convert.cpp
index 5f42676c95..1adabaeb36 100644
--- a/subprojects/gst-plugins-bad/sys/d3d11/gstd3d11convert.cpp
+++ b/subprojects/gst-plugins-bad/sys/d3d11/gstd3d11convert.cpp
@@ -1618,10 +1618,15 @@ gst_d3d11_base_convert_setup_msaa_texture (GstD3D11BaseConvert * self,
     return;
   }
 
-  if (device_format.dxgi_format != DXGI_FORMAT_UNKNOWN)
+  if (device_format.dxgi_format == DXGI_FORMAT_UNKNOWN ||
+      (device_format.format_support[0] &
+          D3D11_FORMAT_SUPPORT_RENDER_TARGET) == 0) {
+    GST_DEBUG_OBJECT (self,
+        "MSAA supported only for native and RTV available formats");;
+    return;
+  } else {
     dxgi_format = device_format.dxgi_format;
-  else
-    dxgi_format = device_format.resource_format[0];
+  }
 
   device_handle = gst_d3d11_device_get_device_handle (device);
   while (sample_count > 1) {