From 6ef0ef51dc1ba58e3fd6929e201e1c2cd15198b6 Mon Sep 17 00:00:00 2001 From: Diego de Souza Date: Mon, 16 Mar 2026 00:14:42 +0100 Subject: [PATCH] avcodec/nvdec: fix surface pool limits and unsafe_output lifetime Cap ulNumDecodeSurfaces to 32 and ulNumOutputSurfaces to 64 to prevent cuvidCreateDecoder from failing with CUDA_ERROR_INVALID_VALUE when initial_pool_size exceeds the hardware limits. Also cap the decoder index pool (dpb_size) to 32 so that indices handed out via av_refstruct_pool_get stay within the valid range for cuvidDecodePicture's CurrPicIdx. When unsafe_output is enabled, stop holding idx_ref in the unmap callback. Since cuvidMapVideoFrame copies decoded data into an independent output mapping slot, the decode surface index can safely be reused as soon as the DPB releases it, without waiting for the downstream consumer to release the mapped frame. This decouples the decode surface index lifetime (max 32) from the output mapping slot lifetime (max 64), eliminating the "No decoder surfaces left" error that occurred when downstream components like nvenc held too many frames. Signed-off-by: Diego de Souza --- libavcodec/nvdec.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/libavcodec/nvdec.c b/libavcodec/nvdec.c index 7c29f25718..60becd2733 100644 --- a/libavcodec/nvdec.c +++ b/libavcodec/nvdec.c @@ -413,8 +413,8 @@ int ff_nvdec_decode_init(AVCodecContext *avctx) params.OutputFormat = output_format; params.CodecType = cuvid_codec_type; params.ChromaFormat = cuvid_chroma_format; - params.ulNumDecodeSurfaces = frames_ctx->initial_pool_size; - params.ulNumOutputSurfaces = unsafe_output ? frames_ctx->initial_pool_size : 1; + params.ulNumDecodeSurfaces = FFMIN(frames_ctx->initial_pool_size, 32); + params.ulNumOutputSurfaces = unsafe_output ? FFMIN(frames_ctx->initial_pool_size, 64) : 1; ret = nvdec_decoder_create(&ctx->decoder, frames_ctx->device_ref, ¶ms, avctx); if (ret < 0) { @@ -438,7 +438,7 @@ int ff_nvdec_decode_init(AVCodecContext *avctx) ret = AVERROR(ENOMEM); goto fail; } - pool->dpb_size = frames_ctx->initial_pool_size; + pool->dpb_size = FFMIN(frames_ctx->initial_pool_size, 32); ctx->decoder_pool = av_refstruct_pool_alloc_ext(sizeof(unsigned int), 0, pool, nvdec_decoder_frame_init, @@ -543,7 +543,6 @@ static int nvdec_retrieve_data(void *logctx, AVFrame *frame) goto copy_fail; unmap_data->idx = cf->idx; - unmap_data->idx_ref = av_refstruct_ref(cf->idx_ref); unmap_data->decoder = av_refstruct_ref(cf->decoder); av_pix_fmt_get_chroma_sub_sample(hwctx->sw_format, &shift_h, &shift_v);