PR #23820 opened by Diego de Souza (ddesouza) URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/23820 Patch URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/23820.patch
Video Codec SDK 13.1 exposes the MV-HEVC picture parameters needed by NVDEC. Populate them from the parsed VPS and inter-layer reference sets, and accept the Multiview Main profile when those fields are available. The HEVC decoder starts all layers before ending any of them, while the NVDEC context stores only one pending picture. Submit a pending layer before the next layer overwrites its parameters and slices. Keep marked frames from every layer in the hardware DPB so NVDEC does not release an enhancement-layer temporal reference during a base-layer submission. Map active reference sets by frame identity because POCs are not unique across layers. Keep active inter-layer references separate from NumPocTotalCurr as required by NVDEC, validate the SDK array limits, and size the surface pool for every decoded layer and frame thread. Warn when the resulting request exceeds NVDEC's 32-surface limit. **Tests** Tested on an NVIDIA GPU by comparing software and NVDEC frame hashes for both views of the multiview.mov and pico-mv-hevc.mp4 FATE samples. All 148 decoded view frames matched. The 128x128 mv_nuh_layer_id.bit sample is below the tested GPU's minimum NVDEC width and was skipped. >From 7d770bf509ef47741933d64f355d6f7f1d2b1668 Mon Sep 17 00:00:00 2001 From: Diego de Souza <[email protected]> Date: Thu, 19 Mar 2026 23:54:37 +0100 Subject: [PATCH] avcodec/nvdec_hevc: add MV-HEVC hardware decoding Video Codec SDK 13.1 exposes the MV-HEVC picture parameters needed by NVDEC. Populate them from the parsed VPS and inter-layer reference sets, and accept the Multiview Main profile when those fields are available. The HEVC decoder starts all layers before ending any of them, while the NVDEC context stores only one pending picture. Submit a pending layer before the next layer overwrites its parameters and slices. Keep marked frames from every layer in the hardware DPB so NVDEC does not release an enhancement-layer temporal reference during a base-layer submission. Map active reference sets by frame identity because POCs are not unique across layers. Keep active inter-layer references separate from NumPocTotalCurr as required by NVDEC, validate the SDK array limits, and size the surface pool for every decoded layer and frame thread. Warn when the resulting request exceeds NVDEC's 32-surface limit. Signed-off-by: Diego de Souza <[email protected]> --- libavcodec/nvdec.c | 16 ++-- libavcodec/nvdec.h | 1 + libavcodec/nvdec_hevc.c | 192 +++++++++++++++++++++++++++++++++------- 3 files changed, 169 insertions(+), 40 deletions(-) diff --git a/libavcodec/nvdec.c b/libavcodec/nvdec.c index 278061d572..f2ac4d0817 100644 --- a/libavcodec/nvdec.c +++ b/libavcodec/nvdec.c @@ -625,19 +625,17 @@ int ff_nvdec_decode_init(AVCodecContext *avctx) params.OutputFormat = output_format; params.CodecType = cuvid_codec_type; params.ChromaFormat = cuvid_chroma_format; + if (decode_pool_size > 32) + av_log(avctx, AV_LOG_WARNING, + "NVDEC requires %d decode surfaces, exceeding the hardware limit " + "of 32; capping to 32. Decoding may fail for this stream.\n", + decode_pool_size); params.ulNumDecodeSurfaces = FFMIN(decode_pool_size, 32); params.ulNumOutputSurfaces = opaque_output ? 0 : (unsafe_output ? FFMIN(frames_ctx->initial_pool_size, 64) : 1); ret = nvdec_decoder_create(&ctx->decoder, frames_ctx->device_ref, ¶ms, avctx); - if (ret < 0) { - if (params.ulNumDecodeSurfaces > 32) { - av_log(avctx, AV_LOG_WARNING, "Using more than 32 (%d) decode surfaces might cause nvdec to fail.\n", - (int)params.ulNumDecodeSurfaces); - av_log(avctx, AV_LOG_WARNING, "Try lowering the amount of threads. Using %d right now.\n", - avctx->thread_count); - } + if (ret < 0) goto fail; - } decoder = ctx->decoder; decoder->unsafe_output = unsafe_output; @@ -662,7 +660,7 @@ int ff_nvdec_decode_init(AVCodecContext *avctx) goto fail; reg_info.ulNumDecodeSurfaces = FFMIN(cuda_priv->cuarray_num_surfaces, - (unsigned)decode_pool_size); + params.ulNumDecodeSurfaces); reg_info.pDecodeSurfaces = cuda_priv->cuarray_surfaces; ret = CHECK_CU(decoder->cvdl->cuvidRegisterDecodeSurfaces(decoder->decoder, ®_info)); CHECK_CU(decoder->cudl->cuCtxPopCurrent(&dummy)); diff --git a/libavcodec/nvdec.h b/libavcodec/nvdec.h index 756192acd2..61a5d53626 100644 --- a/libavcodec/nvdec.h +++ b/libavcodec/nvdec.h @@ -49,6 +49,7 @@ // SDK 13.1 compile time feature checks #if NVDECAPI_CHECK_VERSION(13, 1) #define NVDEC_HAVE_OPAQUE_OUTPUT_SUPPORT +#define NVDEC_HAVE_MVHEVC_DECODE #endif typedef struct NVDECFrame { diff --git a/libavcodec/nvdec_hevc.c b/libavcodec/nvdec_hevc.c index 601ace7d29..ecf47ed7fb 100644 --- a/libavcodec/nvdec_hevc.c +++ b/libavcodec/nvdec_hevc.c @@ -44,6 +44,17 @@ static void dpb_add(CUVIDHEVCPICPARAMS *pp, int idx, const HEVCFrame *src) pp->IsLongTerm[idx] = !!(src->flags & HEVC_FRAME_FLAG_LONG_REF); } +static int dpb_find(const HEVCFrame *const *dpb, int dpb_size, + const HEVCFrame *ref) +{ + for (int i = 0; i < dpb_size; i++) { + if (dpb[i] == ref) + return i; + } + + return -1; +} + static void fill_scaling_lists(CUVIDHEVCPICPARAMS *ppc, const HEVCContext *s) { const ScalingList *sl = s->pps->scaling_list_data_present_flag ? @@ -76,7 +87,6 @@ static int nvdec_hevc_start_frame(AVCodecContext *avctx, const uint8_t *buffer, uint32_t size) { const HEVCContext *s = avctx->priv_data; - const HEVCLayerContext *l = &s->layers[s->cur_layer]; const HEVCPPS *pps = s->pps; const HEVCSPS *sps = pps->sps; @@ -85,8 +95,18 @@ static int nvdec_hevc_start_frame(AVCodecContext *avctx, CUVIDHEVCPICPARAMS *ppc = &pp->CodecSpecific.hevc; FrameDecodeData *fdd; NVDECFrame *cf; + const HEVCFrame *dpb[FF_ARRAY_ELEMS(ppc->RefPicIdx)]; - int i, j, dpb_size, ret; + int i, dpb_size, nb_dpb_layers = 1, ret; + + /* The HEVC decoder starts every layer before ending any of them, while + * NVDECContext has storage for only one pending picture. Submit the + * previous layer before its parameters and slices are overwritten. */ + if (ctx->nb_slices) { + ret = ff_nvdec_end_frame(avctx); + if (ret < 0) + return ret; + } ret = ff_nvdec_start_frame(avctx, s->cur_frame->f); if (ret < 0) @@ -199,6 +219,42 @@ static int nvdec_hevc_start_frame(AVCodecContext *avctx, }, }; +#ifdef NVDEC_HAVE_MVHEVC_DECODE + if (sps->vps && sps->vps->nb_layers > 1) { + const HEVCVPS *vps = sps->vps; + int layer_idx = vps->layer_idx[s->nuh_layer_id]; + int nb_inter_layer_refs = s->rps[INTER_LAYER0].nb_refs + + s->rps[INTER_LAYER1].nb_refs; + + if (layer_idx < 0 || layer_idx >= vps->nb_layers) { + av_log(avctx, AV_LOG_ERROR, + "Invalid MV-HEVC layer ID: %d\n", s->nuh_layer_id); + return AVERROR_INVALIDDATA; + } + if (s->rps[INTER_LAYER0].nb_refs > FF_ARRAY_ELEMS(ppc->RefPicSetInterLayer0) || + s->rps[INTER_LAYER1].nb_refs > FF_ARRAY_ELEMS(ppc->RefPicSetInterLayer1)) { + av_log(avctx, AV_LOG_ERROR, "Too many inter-layer reference frames\n"); + return AVERROR_INVALIDDATA; + } + if (ppc->NumPocTotalCurr < nb_inter_layer_refs) { + av_log(avctx, AV_LOG_ERROR, "Invalid inter-layer reference count\n"); + return AVERROR_INVALIDDATA; + } + + ppc->mv_hevc_enable = 1; + ppc->nuh_layer_id = s->nuh_layer_id; + ppc->default_ref_layers_active_flag = vps->default_ref_layers_active; + ppc->NumDirectRefLayers = vps->num_direct_ref_layers[layer_idx]; + ppc->max_one_active_ref_layer_flag = vps->max_one_active_ref_layer; + ppc->poc_lsb_not_present_flag = (vps->poc_lsb_not_present >> layer_idx) & 1; + ppc->NumActiveRefLayerPics0 = s->rps[INTER_LAYER0].nb_refs; + ppc->NumActiveRefLayerPics1 = s->rps[INTER_LAYER1].nb_refs; + /* NVDEC counts active inter-layer pictures separately. */ + ppc->NumPocTotalCurr -= nb_inter_layer_refs; + nb_dpb_layers = vps->nb_layers; + } +#endif + if (pps->num_tile_columns > FF_ARRAY_ELEMS(ppc->column_width_minus1) || pps->num_tile_rows > FF_ARRAY_ELEMS(ppc->row_height_minus1)) { av_log(avctx, AV_LOG_ERROR, "Too many tiles\n"); @@ -229,43 +285,73 @@ static int nvdec_hevc_start_frame(AVCodecContext *avctx, } dpb_size = 0; - for (i = 0; i < FF_ARRAY_ELEMS(l->DPB); i++) { - const HEVCFrame *ref = &l->DPB[i]; - if (!(ref->flags & (HEVC_FRAME_FLAG_SHORT_REF | HEVC_FRAME_FLAG_LONG_REF))) - continue; - if (dpb_size >= FF_ARRAY_ELEMS(ppc->RefPicIdx)) { - av_log(avctx, AV_LOG_ERROR, "Too many reference frames\n"); + for (int layer = 0; layer < nb_dpb_layers; layer++) { + const HEVCLayerContext *layer_ctx = &s->layers[layer]; + + for (i = 0; i < FF_ARRAY_ELEMS(layer_ctx->DPB); i++) { + const HEVCFrame *ref = &layer_ctx->DPB[i]; + + if (!(ref->flags & (HEVC_FRAME_FLAG_SHORT_REF | + HEVC_FRAME_FLAG_LONG_REF))) + continue; + if (dpb_size >= FF_ARRAY_ELEMS(ppc->RefPicIdx)) { + av_log(avctx, AV_LOG_ERROR, "Too many reference frames\n"); + return AVERROR_INVALIDDATA; + } + dpb[dpb_size] = ref; + dpb_add(ppc, dpb_size++, ref); + } + } + +#ifdef NVDEC_HAVE_MVHEVC_DECODE + /* Inter-layer references are long-term references for MV-HEVC. */ + for (i = 0; i < s->rps[INTER_LAYER0].nb_refs; i++) { + int idx = dpb_find(dpb, dpb_size, s->rps[INTER_LAYER0].ref[i]); + + if (idx < 0) { + av_log(avctx, AV_LOG_ERROR, + "Inter-layer reference frame missing from the DPB\n"); return AVERROR_INVALIDDATA; } - dpb_add(ppc, dpb_size++, ref); - + ppc->IsLongTerm[idx] = 1; + ppc->RefPicSetInterLayer0[i] = idx; } + for (i = 0; i < s->rps[INTER_LAYER1].nb_refs; i++) { + int idx = dpb_find(dpb, dpb_size, s->rps[INTER_LAYER1].ref[i]); + + if (idx < 0) { + av_log(avctx, AV_LOG_ERROR, + "Inter-layer reference frame missing from the DPB\n"); + return AVERROR_INVALIDDATA; + } + ppc->IsLongTerm[idx] = 1; + ppc->RefPicSetInterLayer1[i] = idx; + } +#endif + for (i = dpb_size; i < FF_ARRAY_ELEMS(ppc->RefPicIdx); i++) ppc->RefPicIdx[i] = -1; for (i = 0; i < s->rps[ST_CURR_BEF].nb_refs; i++) { - for (j = 0; j < dpb_size; j++) { - if (ppc->PicOrderCntVal[j] == s->rps[ST_CURR_BEF].list[i]) { - ppc->RefPicSetStCurrBefore[i] = j; - break; - } - } + int idx = dpb_find(dpb, dpb_size, s->rps[ST_CURR_BEF].ref[i]); + + if (idx < 0) + return AVERROR_BUG; + ppc->RefPicSetStCurrBefore[i] = idx; } for (i = 0; i < s->rps[ST_CURR_AFT].nb_refs; i++) { - for (j = 0; j < dpb_size; j++) { - if (ppc->PicOrderCntVal[j] == s->rps[ST_CURR_AFT].list[i]) { - ppc->RefPicSetStCurrAfter[i] = j; - break; - } - } + int idx = dpb_find(dpb, dpb_size, s->rps[ST_CURR_AFT].ref[i]); + + if (idx < 0) + return AVERROR_BUG; + ppc->RefPicSetStCurrAfter[i] = idx; } for (i = 0; i < s->rps[LT_CURR].nb_refs; i++) { - for (j = 0; j < dpb_size; j++) { - if (ppc->PicOrderCntVal[j] == s->rps[LT_CURR].list[i]) { - ppc->RefPicSetLtCurr[i] = j; - break; - } - } + int idx = dpb_find(dpb, dpb_size, s->rps[LT_CURR].ref[i]); + + if (idx < 0) + return AVERROR_BUG; + ppc->RefPicSetLtCurr[i] = idx; } fill_scaling_lists(ppc, s); @@ -300,12 +386,53 @@ static int nvdec_hevc_decode_slice(AVCodecContext *avctx, const uint8_t *buffer, return 0; } +static int nvdec_hevc_end_frame(AVCodecContext *avctx) +{ + NVDECContext *ctx = avctx->internal->hwaccel_priv_data; + int ret; + + if (!ctx->nb_slices) + return 0; + + ret = ff_nvdec_end_frame(avctx); + ctx->bitstream_len = 0; + ctx->nb_slices = 0; + + return ret; +} + static int nvdec_hevc_frame_params(AVCodecContext *avctx, AVBufferRef *hw_frames_ctx) { const HEVCContext *s = avctx->priv_data; const HEVCSPS *sps = s->pps->sps; - return ff_nvdec_frame_params(avctx, hw_frames_ctx, sps->temporal_layer[sps->max_sub_layers - 1].max_dec_pic_buffering + 1, 1); + int dpb_size = sps->temporal_layer[sps->max_sub_layers - 1].max_dec_pic_buffering + 1; + +#ifdef NVDEC_HAVE_MVHEVC_DECODE + if (sps->vps && sps->vps->nb_layers > 1) { + const HEVCVPS *vps = sps->vps; + int nb_layers = vps->nb_layers; + int vps_dpb = FFMAX(vps->vps_max_dec_pic_buffering[vps->vps_max_sub_layers - 1], + vps->dpb_size.max_dec_pic_buffering) + 1; + dpb_size = FFMAX(dpb_size, vps_dpb); + dpb_size *= nb_layers; + + if (avctx->active_thread_type & FF_THREAD_FRAME) { + int max_threads = FFMAX((32 - dpb_size - 2) / nb_layers, 1); + + if (avctx->thread_count > max_threads) + av_log(avctx, AV_LOG_WARNING, + "MV-HEVC NVDEC requires %d surfaces per frame thread; " + "use -threads %d or lower to stay within the 32-surface " + "hardware limit.\n", nb_layers, max_threads); + + /* Generic hwaccel setup adds one surface per frame thread. */ + dpb_size += avctx->thread_count * (nb_layers - 1); + } + } +#endif + + return ff_nvdec_frame_params(avctx, hw_frames_ctx, dpb_size, 1); } static int nvdec_hevc_decode_init(AVCodecContext *avctx) { @@ -315,6 +442,9 @@ static int nvdec_hevc_decode_init(AVCodecContext *avctx) { if (avctx->profile != AV_PROFILE_HEVC_MAIN && avctx->profile != AV_PROFILE_HEVC_MAIN_10 && avctx->profile != AV_PROFILE_HEVC_MAIN_STILL_PICTURE && +#ifdef NVDEC_HAVE_MVHEVC_DECODE + avctx->profile != AV_PROFILE_HEVC_MULTIVIEW_MAIN && +#endif avctx->profile != AV_PROFILE_HEVC_REXT) { av_log(avctx, AV_LOG_ERROR, "Unsupported HEVC profile: %d\n", avctx->profile); return AVERROR(ENOTSUP); @@ -330,7 +460,7 @@ const FFHWAccel ff_hevc_nvdec_hwaccel = { .p.id = AV_CODEC_ID_HEVC, .p.pix_fmt = AV_PIX_FMT_CUDA, .start_frame = nvdec_hevc_start_frame, - .end_frame = ff_nvdec_end_frame, + .end_frame = nvdec_hevc_end_frame, .decode_slice = nvdec_hevc_decode_slice, .frame_params = nvdec_hevc_frame_params, .init = nvdec_hevc_decode_init, @@ -346,7 +476,7 @@ const FFHWAccel ff_hevc_nvdec_cuarray_hwaccel = { .p.id = AV_CODEC_ID_HEVC, .p.pix_fmt = AV_PIX_FMT_CUARRAY, .start_frame = nvdec_hevc_start_frame, - .end_frame = ff_nvdec_end_frame, + .end_frame = nvdec_hevc_end_frame, .decode_slice = nvdec_hevc_decode_slice, .frame_params = nvdec_hevc_frame_params, .init = nvdec_hevc_decode_init, -- 2.52.0 _______________________________________________ ffmpeg-devel mailing list -- [email protected] To unsubscribe send an email to [email protected]
