From 67cafe5999e816594d78b907f02e359b4efbf45e Mon Sep 17 00:00:00 2001
From: Jochen Henneberg <jochen@centricular.com>
Date: Thu, 24 Apr 2025 07:48:12 +0200
Subject: [PATCH] va: Fix H264 profile decision logic

The current logic would choose 'baseline' profiles only in case that
these profiles appear in the list first.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/8882>
---
 subprojects/gst-plugins-bad/sys/va/gstvah264enc.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/subprojects/gst-plugins-bad/sys/va/gstvah264enc.c b/subprojects/gst-plugins-bad/sys/va/gstvah264enc.c
index 82740d0f98..7e9b1985fc 100644
--- a/subprojects/gst-plugins-bad/sys/va/gstvah264enc.c
+++ b/subprojects/gst-plugins-bad/sys/va/gstvah264enc.c
@@ -1417,9 +1417,9 @@ _decide_profile (GstVaH264Enc * self, VAProfile * _profile, guint * _rt_format)
         continue;
     }
 
-    /* baseline only support I/P mode. */
-    if (self->gop.num_bframes > 0) {
-      if (g_strstr_len (profile_name, -1, "baseline"))
+    /* baseline only support I/P mode and neither cabac nor dct8x8. */
+    if (!self->use_dct8x8 && !self->use_cabac && self->gop.num_bframes == 0) {
+      if (!g_strstr_len (profile_name, -1, "baseline"))
         continue;
     }