diff --git a/libavcodec/encode.c b/libavcodec/encode.c
index 5575cf23db..618be0573d 100644
--- a/libavcodec/encode.c
+++ b/libavcodec/encode.c
@@ -247,11 +247,8 @@ end:
     if (ret < 0 || !got_packet)
         av_packet_unref(avpkt);
 
-    if (frame) {
-        if (!ret)
-            avctx->frame_number++;
+    if (frame)
         av_frame_unref(frame);
-    }
 
     if (got_packet)
         // Encoders must always return ref-counted buffers.
@@ -383,6 +380,8 @@ int attribute_align_arg avcodec_send_frame(AVCodecContext *avctx, const AVFrame
             return ret;
     }
 
+    avctx->frame_number++;
+
     return 0;
 }