msdk: encoder: h264: Enable trellis quantization tuning

Add a new property "trellis" to enable trellis quantization.
Keeping trellis as a flag value (which is boolean for gst x264 enc element)
since it is possible to enable/disable this seperately for
I,P and B frames through MediaSDK ext option headers.

The subclass implementations always need to inform base-encoder
if it requires the inclusion of Extend Header buffers (mfxExtCodingOption2
 and mfxExtCodingOption3).

https://bugzilla.gnome.org/show_bug.cgi?id=791637

sys/msdk/gstmsdkh264enc.c

@@ -47,12 +47,16 @@ enum
   PROP_LOW_POWER,
   PROP_FRAME_PACKING,
   PROP_RC_LA_DOWNSAMPLING,
+  PROP_TRELLIS,
 };
 
+#define _MFX_TRELLIS_NONE    0
+
 #define PROP_CABAC_DEFAULT              TRUE
 #define PROP_LOWPOWER_DEFAULT           FALSE
 #define PROP_FRAME_PACKING_DEFAULT      -1
 #define PROP_RC_LA_DOWNSAMPLING_DEFAULT MFX_LOOKAHEAD_DS_UNKNOWN
+#define PROP_TRELLIS_DEFAULT            _MFX_TRELLIS_NONE
 
 static GstStaticPadTemplate src_factory = GST_STATIC_PAD_TEMPLATE ("src",
     GST_PAD_SRC,
@@ -104,6 +108,26 @@ gst_msdkh264enc_rc_lookahead_ds_get_type (void)
   return type;
 }
 
+static GType
+gst_msdkh264enc_trellis_quantization_get_type (void)
+{
+  static GType type = 0;
+
+  static const GFlagsValue values[] = {
+    {_MFX_TRELLIS_NONE, "Disable for all frames", "None"},
+    {MFX_TRELLIS_I, "Enable for I frames", "i"},
+    {MFX_TRELLIS_P, "Enable for P frames", "p"},
+    {MFX_TRELLIS_B, "Enable for B frames", "b"},
+    {0, NULL, NULL}
+  };
+
+  if (!type) {
+    type =
+        g_flags_register_static ("GstMsdkH264EncTrellisQuantization", values);
+  }
+  return type;
+}
+
 #define gst_msdkh264enc_parent_class parent_class
 G_DEFINE_TYPE (GstMsdkH264Enc, gst_msdkh264enc, GST_TYPE_MSDKENC);
 
@@ -307,6 +331,10 @@ gst_msdkh264enc_configure (GstMsdkEnc * encoder)
       encoder->rate_control == MFX_RATECONTROL_LA_HRD ||
       encoder->rate_control == MFX_RATECONTROL_LA_ICQ)
     encoder->option2.LookAheadDS = thiz->lookahead_ds;
+
+  encoder->option2.Trellis = thiz->trellis ? thiz->trellis : MFX_TRELLIS_OFF;
+  encoder->enable_extopt2 = TRUE;
+
   return TRUE;
 }
 
@@ -425,6 +453,9 @@ gst_msdkh264enc_set_property (GObject * object, guint prop_id,
     case PROP_RC_LA_DOWNSAMPLING:
       thiz->lookahead_ds = g_value_get_enum (value);
       break;
+    case PROP_TRELLIS:
+      thiz->trellis = g_value_get_flags (value);
+      break;
     default:
       G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
       break;
@@ -456,6 +487,9 @@ gst_msdkh264enc_get_property (GObject * object, guint prop_id, GValue * value,
     case PROP_RC_LA_DOWNSAMPLING:
       g_value_set_enum (value, thiz->lookahead_ds);
       break;
+    case PROP_TRELLIS:
+      g_value_set_flags (value, thiz->trellis);
+      break;
     default:
       G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
       break;
@@ -508,6 +542,12 @@ gst_msdkh264enc_class_init (GstMsdkH264EncClass * klass)
           PROP_RC_LA_DOWNSAMPLING_DEFAULT,
           G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
 
+  g_object_class_install_property (gobject_class, PROP_TRELLIS,
+      g_param_spec_flags ("trellis", "Trellis",
+          "Enable Trellis Quantization",
+          gst_msdkh264enc_trellis_quantization_get_type (), _MFX_TRELLIS_NONE,
+          G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
+
   gst_element_class_set_static_metadata (element_class,
       "Intel MSDK H264 encoder", "Codec/Encoder/Video",
       "H264 video encoder based on Intel Media SDK",
@@ -522,4 +562,5 @@ gst_msdkh264enc_init (GstMsdkH264Enc * thiz)
   thiz->lowpower = PROP_LOWPOWER_DEFAULT;
   thiz->frame_packing = PROP_FRAME_PACKING_DEFAULT;
   thiz->lookahead_ds = PROP_RC_LA_DOWNSAMPLING_DEFAULT;
+  thiz->trellis = PROP_TRELLIS_DEFAULT;
 }

sys/msdk/gstmsdkh264enc.h

@@ -63,6 +63,7 @@ struct _GstMsdkH264Enc
   gboolean lowpower;
   gint frame_packing;
   guint lookahead_ds;
+  guint trellis;
 };
 
 struct _GstMsdkH264EncClass