diff --git a/validate/gst/qa/gst-qa-monitor-preload.c b/validate/gst/qa/gst-qa-monitor-preload.c
index 6272ef6477..18faa82f9f 100644
--- a/validate/gst/qa/gst-qa-monitor-preload.c
+++ b/validate/gst/qa/gst-qa-monitor-preload.c
@@ -40,10 +40,9 @@ gst_qa_preload_wrap (GstElement * element)
 
   /* TODO this will actually never unref the runner as it holds a ref
    * to the element */
-  g_object_set_data_full ((GObject *) element, "qa-runner", runner,
-      g_object_unref);
-
-  gst_qa_runner_setup (runner);
+  if (runner)
+    g_object_set_data_full ((GObject *) element, "qa-runner", runner,
+        g_object_unref);
 }
 
 GstElement *
diff --git a/validate/gst/qa/gst-qa-runner.c b/validate/gst/qa/gst-qa-runner.c
index ed995e02b7..2a206cc3b8 100644
--- a/validate/gst/qa/gst-qa-runner.c
+++ b/validate/gst/qa/gst-qa-runner.c
@@ -51,6 +51,8 @@ enum
 
 static guint _signals[LAST_SIGNAL] = { 0 };
 
+static gboolean gst_qa_runner_setup (GstQaRunner * runner);
+
 static void
 gst_qa_runner_dispose (GObject * object)
 {
@@ -119,21 +121,22 @@ gst_qa_runner_new (GstElement * pipeline)
   runner = g_object_new (GST_TYPE_QA_RUNNER, NULL);
   runner->pipeline = gst_object_ref (pipeline);
 
-
   if ((scenario_name = g_getenv ("GST_QA_SCENARIO")))
     runner->scenario = gst_qa_scenario_factory_create (runner, scenario_name);
 
   g_object_set_data ((GObject *) pipeline, "qa-runner", runner);
 
+  if (!gst_qa_runner_setup (runner)) {
+    gst_object_unref (runner);
+    runner = NULL;
+  }
+
   return runner;
 }
 
-gboolean
+static gboolean
 gst_qa_runner_setup (GstQaRunner * runner)
 {
-  if (runner->setup)
-    return TRUE;
-
   GST_INFO_OBJECT (runner, "Starting QA Runner setup");
   runner->monitor =
       gst_qa_monitor_factory_create (GST_OBJECT_CAST (runner->pipeline), runner,
@@ -143,7 +146,6 @@ gst_qa_runner_setup (GstQaRunner * runner)
     return FALSE;
   }
 
-  runner->setup = TRUE;
   GST_DEBUG_OBJECT (runner, "Setup successful");
   return TRUE;
 }
diff --git a/validate/gst/qa/gst-qa-runner.h b/validate/gst/qa/gst-qa-runner.h
index 38b867f11d..8fe10fff6b 100644
--- a/validate/gst/qa/gst-qa-runner.h
+++ b/validate/gst/qa/gst-qa-runner.h
@@ -80,7 +80,6 @@ struct _GstQaRunnerClass {
 GType		gst_qa_runner_get_type		(void);
 
 GstQaRunner *   gst_qa_runner_new               (GstElement * pipeline);
-gboolean        gst_qa_runner_setup             (GstQaRunner * runner);
 
 void            gst_qa_runner_add_report  (GstQaRunner * runner, GstQaReport * report);
 void            gst_qa_runner_print_reports (GstQaRunner * runner);
diff --git a/validate/gst/qa/gst-qa-transcoding.c b/validate/gst/qa/gst-qa-transcoding.c
index 6ca4c0738b..ad24eda6eb 100644
--- a/validate/gst/qa/gst-qa-transcoding.c
+++ b/validate/gst/qa/gst-qa-transcoding.c
@@ -300,7 +300,7 @@ main (int argc, gchar ** argv)
   runner = gst_qa_runner_new (pipeline);
   mainloop = g_main_loop_new (NULL, FALSE);
 
-  if (!gst_qa_runner_setup (runner)) {
+  if (!runner) {
     g_printerr ("Failed to setup QA Runner\n");
     exit (1);
   }
diff --git a/validate/gst/qa/gst-qa.c b/validate/gst/qa/gst-qa.c
index 7a732a2226..e510d1580b 100644
--- a/validate/gst/qa/gst-qa.c
+++ b/validate/gst/qa/gst-qa.c
@@ -89,7 +89,7 @@ main (int argc, gchar ** argv)
   runner = gst_qa_runner_new (pipeline);
   mainloop = g_main_loop_new (NULL, FALSE);
 
-  if (!gst_qa_runner_setup (runner)) {
+  if (!runner) {
     g_printerr ("Failed to setup QA Runner\n");
     exit (1);
   }