diff --git a/gst-libs/gst/video/gstvideoencoder.c b/gst-libs/gst/video/gstvideoencoder.c
index 933bb9bf7f..5e5527c5fd 100644
--- a/gst-libs/gst/video/gstvideoencoder.c
+++ b/gst-libs/gst/video/gstvideoencoder.c
@@ -1185,6 +1185,7 @@ gst_video_encoder_chain (GstPad * pad, GstObject * parent, GstBuffer * buf)
   GST_LOG_OBJECT (encoder, "passing frame pfn %d to subclass",
       frame->presentation_frame_number);
 
+  gst_video_codec_frame_ref (frame);
   ret = klass->handle_frame (encoder, frame);
 
 done:
@@ -1496,7 +1497,9 @@ gst_video_encoder_finish_frame (GstVideoEncoder * encoder,
 done:
   /* handed out */
   priv->frames = g_list_remove (priv->frames, frame);
-
+  /* Remove the reference from the list and the reference that
+   * was provided to us */
+  gst_video_codec_frame_unref (frame);
   gst_video_codec_frame_unref (frame);
 
   GST_VIDEO_ENCODER_STREAM_UNLOCK (encoder);