diff --git a/ffmpeg.c b/ffmpeg.c
index bd75d4063d..a37bcd814b 100644
--- a/ffmpeg.c
+++ b/ffmpeg.c
@@ -1155,7 +1155,10 @@ static void do_video_out(AVFormatContext *s,
     if (!ost->last_frame)
         ost->last_frame = av_frame_alloc();
     av_frame_unref(ost->last_frame);
-    av_frame_ref(ost->last_frame, next_picture);
+    if (next_picture)
+        av_frame_ref(ost->last_frame, next_picture);
+    else
+        av_frame_free(&ost->last_frame);
 }
 
 static double psnr(double d)