diff --git a/gst/librfb/gstrfbsrc.c b/gst/librfb/gstrfbsrc.c
index e5d213b242..9db0bd6f72 100644
--- a/gst/librfb/gstrfbsrc.c
+++ b/gst/librfb/gstrfbsrc.c
@@ -201,7 +201,6 @@ gst_rfb_src_finalize (GObject * object)
   }
   if (src->decoder) {
     rfb_decoder_free (src->decoder);
-    g_free (src->decoder);
     src->decoder = NULL;
   }
 
@@ -434,7 +433,6 @@ gst_rfb_src_start (GstBaseSrc * bsrc)
   if (!rfb_decoder_connect_tcp (decoder, src->host, src->port)) {
     GST_ELEMENT_ERROR (src, RESOURCE, READ, (NULL),
         ("Could not connect to host %s on port %d", src->host, src->port));
-    rfb_decoder_free (decoder);
     return FALSE;
   }
 
diff --git a/gst/librfb/rfbdecoder.c b/gst/librfb/rfbdecoder.c
index edc18ec112..3677b345b3 100644
--- a/gst/librfb/rfbdecoder.c
+++ b/gst/librfb/rfbdecoder.c
@@ -88,6 +88,8 @@ rfb_decoder_free (RfbDecoder * decoder)
 
   if (decoder->data)
     g_free (decoder->data);
+
+  g_free (decoder);
 }
 
 gboolean