From c3ebfcd6e1327ca7bbcaee822e593c2da6cfd352 Mon Sep 17 00:00:00 2001 From: Hendrik Leppkes Date: Mon, 18 Mar 2013 18:54:44 +0100 Subject: [PATCH] mpegvideo: allocate hwaccel privdata after the frame buffer This ensures the hwaccel privdata does not leak when a frame buffer could not be allocated (and toggle the assert when the frame is re-used). Having no frame buffer available is quite common when using the DXVA2 hwaccel in situations where the DXVA2 renderer is being re-allocated, for example when moving between displays. Signed-off-by: Anton Khirnov --- libavcodec/mpegvideo.c | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/libavcodec/mpegvideo.c b/libavcodec/mpegvideo.c index ae6cdefe6b..dab1485fcc 100644 --- a/libavcodec/mpegvideo.c +++ b/libavcodec/mpegvideo.c @@ -266,18 +266,6 @@ static int alloc_frame_buffer(MpegEncContext *s, Picture *pic) { int r, ret; - if (s->avctx->hwaccel) { - assert(!pic->hwaccel_picture_private); - if (s->avctx->hwaccel->priv_data_size) { - pic->hwaccel_priv_buf = av_buffer_allocz(s->avctx->hwaccel->priv_data_size); - if (!pic->hwaccel_priv_buf) { - av_log(s->avctx, AV_LOG_ERROR, "alloc_frame_buffer() failed (hwaccel private data allocation)\n"); - return -1; - } - pic->hwaccel_picture_private = pic->hwaccel_priv_buf->data; - } - } - pic->tf.f = &pic->f; if (s->codec_id != AV_CODEC_ID_WMV3IMAGE && s->codec_id != AV_CODEC_ID_VC1IMAGE && @@ -294,11 +282,21 @@ static int alloc_frame_buffer(MpegEncContext *s, Picture *pic) if (r < 0 || !pic->f.data[0]) { av_log(s->avctx, AV_LOG_ERROR, "get_buffer() failed (%d %p)\n", r, pic->f.data[0]); - av_buffer_unref(&pic->hwaccel_priv_buf); - pic->hwaccel_picture_private = NULL; return -1; } + if (s->avctx->hwaccel) { + assert(!pic->hwaccel_picture_private); + if (s->avctx->hwaccel->priv_data_size) { + pic->hwaccel_priv_buf = av_buffer_allocz(s->avctx->hwaccel->priv_data_size); + if (!pic->hwaccel_priv_buf) { + av_log(s->avctx, AV_LOG_ERROR, "alloc_frame_buffer() failed (hwaccel private data allocation)\n"); + return -1; + } + pic->hwaccel_picture_private = pic->hwaccel_priv_buf->data; + } + } + if (s->linesize && (s->linesize != pic->f.linesize[0] || s->uvlinesize != pic->f.linesize[1])) { av_log(s->avctx, AV_LOG_ERROR,