PR #25231 opened by michaelni URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/25231 Patch URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/25231.patch
>From f534edfeb680ea9001616b161dcd07252dbcabfa Mon Sep 17 00:00:00 2001 From: Michael Niedermayer <[email protected]> Date: Sat, 17 Oct 2026 03:18:25 +0200 Subject: [PATCH 1/8] avcodec/nvdec: forget the previous picture's bitstream pointer when a picture starts Fixes: I8tidri5usIx Fixes: AISLE-2026-0150-00880 Fixes: use after free Use after free Replicated through UnModified FFmpeg with ASAN Found-by: Joshua Rogers (AISLE) --- libavcodec/nvdec.c | 1 + 1 file changed, 1 insertion(+) diff --git a/libavcodec/nvdec.c b/libavcodec/nvdec.c index d2c176e4f97..2f88df65c99 100644 --- a/libavcodec/nvdec.c +++ b/libavcodec/nvdec.c @@ -972,6 +972,7 @@ int ff_nvdec_start_frame(AVCodecContext *avctx, AVFrame *frame) NVDECFrame *cf = NULL; int ret; + ctx->bitstream = NULL; ctx->bitstream_len = 0; ctx->nb_slices = 0; -- 2.54.0 >From d37261710807860fcfdb8780e8c31e1fbdabaaf0 Mon Sep 17 00:00:00 2001 From: Lynne <[email protected]> Date: Thu, 18 Dec 2025 01:34:58 +0100 Subject: [PATCH 2/8] mjpegdec: use refstruct for the hwaccel_picture_private This was missed when the hwaccel_picture_private was made a buffer. Fixes: YMGIFa63VNkS Fixes: AISLE-2026-0150-00681 Fixes: memleak Memleak Replicated through UnModified FFmpeg with ASAN Found-by: Joshua Rogers (AISLE) Cherry-picked from https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/24652 where the commit is part of the Vulkan MJPEG hwaccel. The Fixes, replication and Found-by lines were added when applying it for the security report; the start_frame() call keeps passing NULL as the preceding commits of that pull request are not included. --- libavcodec/mjpegdec.c | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/libavcodec/mjpegdec.c b/libavcodec/mjpegdec.c index cbb7268ed92..b0a65f49cec 100644 --- a/libavcodec/mjpegdec.c +++ b/libavcodec/mjpegdec.c @@ -823,15 +823,21 @@ int ff_mjpeg_decode_sof(MJpegDecodeContext *s) if (s->avctx->hwaccel) { const FFHWAccel *hwaccel = ffhwaccel(s->avctx->hwaccel); - s->hwaccel_picture_private = - av_mallocz(hwaccel->frame_priv_data_size); - if (!s->hwaccel_picture_private) - return AVERROR(ENOMEM); + + /* A previous frame may have been aborted before its end_frame call, + * leaving the private data behind. */ + av_refstruct_unref(&s->hwaccel_picture_private); + + ret = ff_hwaccel_frame_priv_alloc(s->avctx, &s->hwaccel_picture_private); + if (ret < 0) + return ret; ret = hwaccel->start_frame(s->avctx, NULL, s->raw_image_buffer, s->raw_image_buffer_size); - if (ret < 0) + if (ret < 0) { + av_refstruct_unref(&s->hwaccel_picture_private); return ret; + } } return 0; @@ -2548,7 +2554,7 @@ eoi_parser: if (ret < 0) return ret; - av_freep(&s->hwaccel_picture_private); + av_refstruct_unref(&s->hwaccel_picture_private); } if ((ret = av_frame_ref(frame, s->picture_ptr)) < 0) return ret; @@ -2917,7 +2923,7 @@ av_cold int ff_mjpeg_decode_end(AVCodecContext *avctx) reset_icc_profile(s); - av_freep(&s->hwaccel_picture_private); + av_refstruct_unref(&s->hwaccel_picture_private); av_freep(&s->jls_state); return 0; @@ -2928,6 +2934,8 @@ static av_cold void decode_flush(AVCodecContext *avctx) MJpegDecodeContext *s = avctx->priv_data; s->got_picture = 0; + av_refstruct_unref(&s->hwaccel_picture_private); + s->smv_next_frame = 0; av_frame_unref(s->smv_frame); } -- 2.54.0 >From bc8308e785323fec4b56eb12e84f28d71bdcb064 Mon Sep 17 00:00:00 2001 From: Michael Niedermayer <[email protected]> Date: Sat, 17 Oct 2026 03:18:25 +0200 Subject: [PATCH 3/8] avcodec/mjpegdec: reject a SOF without a preceding SOI when a hwaccel is used Fixes: KICdwlv5dwyY Fixes: AISLE-2026-0150-00792 Fixes: use after free Use after free Replicated through Modified FFmpeg with ASAN Found-by: Joshua Rogers (AISLE) The report suggested clearing raw_image_buffer per packet and rejecting a SOF that is not preceded by a SOI. --- libavcodec/mjpegdec.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/libavcodec/mjpegdec.c b/libavcodec/mjpegdec.c index b0a65f49cec..1f6b76cf1bd 100644 --- a/libavcodec/mjpegdec.c +++ b/libavcodec/mjpegdec.c @@ -823,6 +823,10 @@ int ff_mjpeg_decode_sof(MJpegDecodeContext *s) if (s->avctx->hwaccel) { const FFHWAccel *hwaccel = ffhwaccel(s->avctx->hwaccel); + if (!s->raw_image_buffer) { + av_log(s->avctx, AV_LOG_ERROR, "SOF without SOI\n"); + return AVERROR_INVALIDDATA; + } /* A previous frame may have been aborted before its end_frame call, * leaving the private data behind. */ @@ -2402,6 +2406,8 @@ int ff_mjpeg_decode_frame_from_buf(AVCodecContext *avctx, AVFrame *frame, s->total_ls_decoded_height = 0; s->buf_size = buf_size; + s->raw_image_buffer = NULL; + s->raw_image_buffer_size = 0; av_exif_free(&s->exif_metadata); av_freep(&s->stereo3d); -- 2.54.0 >From 20c6a15fa3cae470cbf34c6a303bb3cb2d57c8c9 Mon Sep 17 00:00:00 2001 From: Michael Niedermayer <[email protected]> Date: Sat, 17 Oct 2026 03:20:00 +0200 Subject: [PATCH 4/8] cosmetics: avcodec/vaapi_hevc: put the common picture first in VAAPIDecodePictureHEVC --- libavcodec/vaapi_hevc.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libavcodec/vaapi_hevc.c b/libavcodec/vaapi_hevc.c index 878f7965c43..3d0f6632fd3 100644 --- a/libavcodec/vaapi_hevc.c +++ b/libavcodec/vaapi_hevc.c @@ -32,6 +32,8 @@ #include "hevc/hevcdec.h" typedef struct VAAPIDecodePictureHEVC { + VAAPIDecodePicture pic; + #if VA_CHECK_VERSION(1, 2, 0) VAPictureParameterBufferHEVCExtension pic_param; VASliceParameterBufferHEVCExtension last_slice_param; @@ -41,8 +43,6 @@ typedef struct VAAPIDecodePictureHEVC { #endif const uint8_t *last_buffer; size_t last_size; - - VAAPIDecodePicture pic; } VAAPIDecodePictureHEVC; static void init_vaapi_pic(VAPictureHEVC *va_pic) -- 2.54.0 >From 9298e58dd9001976ef62198bd9c56af2fa3e4711 Mon Sep 17 00:00:00 2001 From: Michael Niedermayer <[email protected]> Date: Sat, 17 Oct 2026 03:22:33 +0200 Subject: [PATCH 5/8] avcodec/decode: allow a hwaccel free_frame_priv callback without a hw_frames_ctx Found during triage/review of the security report AISLE-2026-R0043 --- libavcodec/decode.c | 9 ++++----- libavcodec/hwaccel_internal.h | 3 ++- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/libavcodec/decode.c b/libavcodec/decode.c index 96da70fb7e6..9e34d44c18b 100644 --- a/libavcodec/decode.c +++ b/libavcodec/decode.c @@ -2349,14 +2349,13 @@ int ff_hwaccel_frame_priv_alloc(AVCodecContext *avctx, void **hwaccel_picture_pr av_assert0(!*hwaccel_picture_private); if (hwaccel->free_frame_priv) { - AVHWFramesContext *frames_ctx; + AVHWDeviceContext *device_ctx = NULL; - if (!avctx->hw_frames_ctx) - return AVERROR(EINVAL); + if (avctx->hw_frames_ctx) + device_ctx = ((AVHWFramesContext *)avctx->hw_frames_ctx->data)->device_ctx; - frames_ctx = (AVHWFramesContext *) avctx->hw_frames_ctx->data; *hwaccel_picture_private = av_refstruct_alloc_ext(hwaccel->frame_priv_data_size, 0, - frames_ctx->device_ctx, + device_ctx, hwaccel->free_frame_priv); } else { *hwaccel_picture_private = av_refstruct_allocz(hwaccel->frame_priv_data_size); diff --git a/libavcodec/hwaccel_internal.h b/libavcodec/hwaccel_internal.h index 7224406815a..ac860f8ab0e 100644 --- a/libavcodec/hwaccel_internal.h +++ b/libavcodec/hwaccel_internal.h @@ -154,7 +154,8 @@ typedef struct FFHWAccel { /** * Callback to free the hwaccel-specific frame data. * - * @param hwctx a pointer to an AVHWDeviceContext. + * @param hwctx a pointer to an AVHWDeviceContext, NULL when the decoder + * has no hw_frames_ctx. * @param data the per-frame hardware accelerator private data to be freed. */ void (*free_frame_priv)(AVRefStructOpaque hwctx, void *data); -- 2.54.0 >From e057f7fcb547eb333afc5d211edbeff171807d02 Mon Sep 17 00:00:00 2001 From: Michael Niedermayer <[email protected]> Date: Sat, 17 Oct 2026 03:22:33 +0200 Subject: [PATCH 6/8] avcodec/vdpau: free the bitstream buffer table when the picture is released Fixes: pzezUDNFoag5 Fixes: AISLE-2026-0150-00646 Fixes: memleak Memleak Replicated through UnModified FFmpeg with ASAN Found-by: Joshua Rogers (AISLE) The report suggested a free_frame_priv callback that frees bitstream_buffers. --- libavcodec/vdpau.c | 7 +++++++ libavcodec/vdpau_av1.c | 1 + libavcodec/vdpau_h264.c | 1 + libavcodec/vdpau_hevc.c | 1 + libavcodec/vdpau_internal.h | 2 ++ libavcodec/vdpau_mpeg12.c | 2 ++ libavcodec/vdpau_mpeg4.c | 1 + libavcodec/vdpau_vc1.c | 2 ++ libavcodec/vdpau_vp9.c | 1 + 9 files changed, 18 insertions(+) diff --git a/libavcodec/vdpau.c b/libavcodec/vdpau.c index e99ac5338dc..2c131135515 100644 --- a/libavcodec/vdpau.c +++ b/libavcodec/vdpau.c @@ -349,6 +349,13 @@ int ff_vdpau_common_end_frame(AVCodecContext *avctx, AVFrame *frame, return vdpau_error(status); } +void ff_vdpau_free_frame_priv(AVRefStructOpaque hwctx, void *data) +{ + struct vdpau_picture_context *pic_ctx = data; + + av_freep(&pic_ctx->bitstream_buffers); +} + #if CONFIG_MPEG1_VDPAU_HWACCEL || \ CONFIG_MPEG2_VDPAU_HWACCEL || CONFIG_MPEG4_VDPAU_HWACCEL || \ CONFIG_VC1_VDPAU_HWACCEL || CONFIG_WMV3_VDPAU_HWACCEL diff --git a/libavcodec/vdpau_av1.c b/libavcodec/vdpau_av1.c index b7e7fa8b0c8..aaf8cd57c0b 100644 --- a/libavcodec/vdpau_av1.c +++ b/libavcodec/vdpau_av1.c @@ -362,6 +362,7 @@ const FFHWAccel ff_av1_vdpau_hwaccel = { .end_frame = vdpau_av1_end_frame, .decode_slice = vdpau_av1_decode_slice, .frame_priv_data_size = sizeof(struct vdpau_picture_context), + .free_frame_priv = ff_vdpau_free_frame_priv, .init = vdpau_av1_init, .uninit = ff_vdpau_common_uninit, .frame_params = ff_vdpau_common_frame_params, diff --git a/libavcodec/vdpau_h264.c b/libavcodec/vdpau_h264.c index 51c7e999007..df0ae7d5c89 100644 --- a/libavcodec/vdpau_h264.c +++ b/libavcodec/vdpau_h264.c @@ -271,6 +271,7 @@ const FFHWAccel ff_h264_vdpau_hwaccel = { .end_frame = vdpau_h264_end_frame, .decode_slice = vdpau_h264_decode_slice, .frame_priv_data_size = sizeof(struct vdpau_picture_context), + .free_frame_priv = ff_vdpau_free_frame_priv, .init = vdpau_h264_init, .uninit = ff_vdpau_common_uninit, .frame_params = ff_vdpau_common_frame_params, diff --git a/libavcodec/vdpau_hevc.c b/libavcodec/vdpau_hevc.c index 1a92c32b27f..1a8429475e8 100644 --- a/libavcodec/vdpau_hevc.c +++ b/libavcodec/vdpau_hevc.c @@ -551,6 +551,7 @@ const FFHWAccel ff_hevc_vdpau_hwaccel = { .end_frame = vdpau_hevc_end_frame, .decode_slice = vdpau_hevc_decode_slice, .frame_priv_data_size = sizeof(struct vdpau_picture_context), + .free_frame_priv = ff_vdpau_free_frame_priv, .init = vdpau_hevc_init, .uninit = ff_vdpau_common_uninit, .frame_params = ff_vdpau_common_frame_params, diff --git a/libavcodec/vdpau_internal.h b/libavcodec/vdpau_internal.h index 347576e0c3b..8a4ed13d1ab 100644 --- a/libavcodec/vdpau_internal.h +++ b/libavcodec/vdpau_internal.h @@ -30,6 +30,7 @@ #include "libavutil/frame.h" #include "libavutil/hwcontext.h" #include "libavutil/hwcontext_vdpau.h" +#include "libavutil/refstruct.h" #include "avcodec.h" #include "vdpau.h" @@ -125,6 +126,7 @@ int ff_vdpau_common_start_frame(struct vdpau_picture_context *pic, const uint8_t *buffer, uint32_t size); int ff_vdpau_common_end_frame(AVCodecContext *avctx, AVFrame *frame, struct vdpau_picture_context *pic); +void ff_vdpau_free_frame_priv(AVRefStructOpaque hwctx, void *data); int ff_vdpau_mpeg_end_frame(AVCodecContext *avctx); int ff_vdpau_add_buffer(struct vdpau_picture_context *pic, const uint8_t *buf, uint32_t buf_size); diff --git a/libavcodec/vdpau_mpeg12.c b/libavcodec/vdpau_mpeg12.c index b3ac94f2172..59047d699c0 100644 --- a/libavcodec/vdpau_mpeg12.c +++ b/libavcodec/vdpau_mpeg12.c @@ -116,6 +116,7 @@ const FFHWAccel ff_mpeg1_vdpau_hwaccel = { .end_frame = ff_vdpau_mpeg_end_frame, .decode_slice = vdpau_mpeg_decode_slice, .frame_priv_data_size = sizeof(struct vdpau_picture_context), + .free_frame_priv = ff_vdpau_free_frame_priv, .init = vdpau_mpeg1_init, .uninit = ff_vdpau_common_uninit, .priv_data_size = sizeof(VDPAUContext), @@ -151,6 +152,7 @@ const FFHWAccel ff_mpeg2_vdpau_hwaccel = { .end_frame = ff_vdpau_mpeg_end_frame, .decode_slice = vdpau_mpeg_decode_slice, .frame_priv_data_size = sizeof(struct vdpau_picture_context), + .free_frame_priv = ff_vdpau_free_frame_priv, .init = vdpau_mpeg2_init, .uninit = ff_vdpau_common_uninit, .frame_params = ff_vdpau_common_frame_params, diff --git a/libavcodec/vdpau_mpeg4.c b/libavcodec/vdpau_mpeg4.c index 58c8a6d7696..10f89805672 100644 --- a/libavcodec/vdpau_mpeg4.c +++ b/libavcodec/vdpau_mpeg4.c @@ -122,6 +122,7 @@ const FFHWAccel ff_mpeg4_vdpau_hwaccel = { .end_frame = ff_vdpau_mpeg_end_frame, .decode_slice = vdpau_mpeg4_decode_slice, .frame_priv_data_size = sizeof(struct vdpau_picture_context), + .free_frame_priv = ff_vdpau_free_frame_priv, .init = vdpau_mpeg4_init, .uninit = ff_vdpau_common_uninit, .frame_params = ff_vdpau_common_frame_params, diff --git a/libavcodec/vdpau_vc1.c b/libavcodec/vdpau_vc1.c index 323ac4127a0..90ce5ace9f7 100644 --- a/libavcodec/vdpau_vc1.c +++ b/libavcodec/vdpau_vc1.c @@ -149,6 +149,7 @@ const FFHWAccel ff_wmv3_vdpau_hwaccel = { .end_frame = ff_vdpau_mpeg_end_frame, .decode_slice = vdpau_vc1_decode_slice, .frame_priv_data_size = sizeof(struct vdpau_picture_context), + .free_frame_priv = ff_vdpau_free_frame_priv, .init = vdpau_vc1_init, .uninit = ff_vdpau_common_uninit, .frame_params = ff_vdpau_common_frame_params, @@ -166,6 +167,7 @@ const FFHWAccel ff_vc1_vdpau_hwaccel = { .end_frame = ff_vdpau_mpeg_end_frame, .decode_slice = vdpau_vc1_decode_slice, .frame_priv_data_size = sizeof(struct vdpau_picture_context), + .free_frame_priv = ff_vdpau_free_frame_priv, .init = vdpau_vc1_init, .uninit = ff_vdpau_common_uninit, .frame_params = ff_vdpau_common_frame_params, diff --git a/libavcodec/vdpau_vp9.c b/libavcodec/vdpau_vp9.c index 7b3c1ff4d7a..83a17ed74d2 100644 --- a/libavcodec/vdpau_vp9.c +++ b/libavcodec/vdpau_vp9.c @@ -232,6 +232,7 @@ const FFHWAccel ff_vp9_vdpau_hwaccel = { .end_frame = vdpau_vp9_end_frame, .decode_slice = vdpau_vp9_decode_slice, .frame_priv_data_size = sizeof(struct vdpau_picture_context), + .free_frame_priv = ff_vdpau_free_frame_priv, .init = vdpau_vp9_init, .uninit = ff_vdpau_common_uninit, .frame_params = ff_vdpau_common_frame_params, -- 2.54.0 >From 18009a8d78f04e53dba36630af178da045b3f6d0 Mon Sep 17 00:00:00 2001 From: Michael Niedermayer <[email protected]> Date: Sat, 17 Oct 2026 03:22:33 +0200 Subject: [PATCH 7/8] avcodec/vaapi_decode: destroy the buffers of a picture released without being issued Fixes: vMMlgSzb6MuU Fixes: AISLE-2026-0150-00741 Fixes: memleak Memleak Replicated through UnModified FFmpeg with ASAN Found-by: Joshua Rogers (AISLE) The report suggested a free_frame_priv callback that destroys the still tracked parameter and slice buffers. --- libavcodec/vaapi_av1.c | 1 + libavcodec/vaapi_decode.c | 47 +++++++++++++++++++++++++-------------- libavcodec/vaapi_decode.h | 2 ++ libavcodec/vaapi_h264.c | 1 + libavcodec/vaapi_hevc.c | 1 + libavcodec/vaapi_mjpeg.c | 1 + libavcodec/vaapi_mpeg2.c | 1 + libavcodec/vaapi_mpeg4.c | 2 ++ libavcodec/vaapi_vc1.c | 2 ++ libavcodec/vaapi_vp8.c | 1 + libavcodec/vaapi_vp9.c | 1 + libavcodec/vaapi_vvc.c | 1 + 12 files changed, 44 insertions(+), 17 deletions(-) diff --git a/libavcodec/vaapi_av1.c b/libavcodec/vaapi_av1.c index d00bb601a53..02fe5246e4d 100644 --- a/libavcodec/vaapi_av1.c +++ b/libavcodec/vaapi_av1.c @@ -456,6 +456,7 @@ const FFHWAccel ff_av1_vaapi_hwaccel = { .end_frame = vaapi_av1_end_frame, .decode_slice = vaapi_av1_decode_slice, .frame_priv_data_size = sizeof(VAAPIDecodePicture), + .free_frame_priv = ff_vaapi_decode_free_frame_priv, .init = vaapi_av1_decode_init, .uninit = vaapi_av1_decode_uninit, .frame_params = ff_vaapi_common_frame_params, diff --git a/libavcodec/vaapi_decode.c b/libavcodec/vaapi_decode.c index 4ac2df5b431..e7f5545dba9 100644 --- a/libavcodec/vaapi_decode.c +++ b/libavcodec/vaapi_decode.c @@ -135,34 +135,44 @@ int ff_vaapi_decode_make_slice_buffer(AVCodecContext *avctx, return 0; } -static void ff_vaapi_decode_destroy_buffers(AVCodecContext *avctx, +static void ff_vaapi_decode_destroy_buffers(void *logctx, VADisplay display, VAAPIDecodePicture *pic) { - VAAPIDecodeContext *ctx = avctx->internal->hwaccel_priv_data; VAStatus vas; int i; for (i = 0; i < pic->nb_param_buffers; i++) { - vas = vaDestroyBuffer(ctx->hwctx->display, - pic->param_buffers[i]); + vas = vaDestroyBuffer(display, pic->param_buffers[i]); if (vas != VA_STATUS_SUCCESS) { - av_log(avctx, AV_LOG_ERROR, "Failed to destroy " + av_log(logctx, AV_LOG_ERROR, "Failed to destroy " "parameter buffer %#x: %d (%s).\n", pic->param_buffers[i], vas, vaErrorStr(vas)); } } for (i = 0; i < 2 * pic->nb_slices; i++) { - vas = vaDestroyBuffer(ctx->hwctx->display, - pic->slice_buffers[i]); + vas = vaDestroyBuffer(display, pic->slice_buffers[i]); if (vas != VA_STATUS_SUCCESS) { - av_log(avctx, AV_LOG_ERROR, "Failed to destroy slice " + av_log(logctx, AV_LOG_ERROR, "Failed to destroy slice " "slice buffer %#x: %d (%s).\n", pic->slice_buffers[i], vas, vaErrorStr(vas)); } } } +static void vaapi_decode_free_buffers(void *logctx, VADisplay display, + VAAPIDecodePicture *pic) +{ + ff_vaapi_decode_destroy_buffers(logctx, display, pic); + + pic->nb_param_buffers = 0; + pic->nb_param_buffers_allocated = 0; + av_freep(&pic->param_buffers); + pic->nb_slices = 0; + pic->nb_slice_buffers_allocated = 0; + av_freep(&pic->slice_buffers); +} + int ff_vaapi_decode_issue(AVCodecContext *avctx, VAAPIDecodePicture *pic) { @@ -219,7 +229,7 @@ int ff_vaapi_decode_issue(AVCodecContext *avctx, if (CONFIG_VAAPI_1 || ctx->hwctx->driver_quirks & AV_VAAPI_DRIVER_QUIRK_RENDER_PARAM_BUFFERS) - ff_vaapi_decode_destroy_buffers(avctx, pic); + ff_vaapi_decode_destroy_buffers(avctx, ctx->hwctx->display, pic); err = 0; goto exit; @@ -231,7 +241,7 @@ fail_with_picture: "after error: %d (%s).\n", vas, vaErrorStr(vas)); } fail: - ff_vaapi_decode_destroy_buffers(avctx, pic); + ff_vaapi_decode_destroy_buffers(avctx, ctx->hwctx->display, pic); fail_at_end: exit: pic->nb_param_buffers = 0; @@ -247,18 +257,21 @@ exit: int ff_vaapi_decode_cancel(AVCodecContext *avctx, VAAPIDecodePicture *pic) { - ff_vaapi_decode_destroy_buffers(avctx, pic); + VAAPIDecodeContext *ctx = avctx->internal->hwaccel_priv_data; - pic->nb_param_buffers = 0; - pic->nb_param_buffers_allocated = 0; - av_freep(&pic->param_buffers); - pic->nb_slices = 0; - pic->nb_slice_buffers_allocated = 0; - av_freep(&pic->slice_buffers); + vaapi_decode_free_buffers(avctx, ctx->hwctx->display, pic); return 0; } +void ff_vaapi_decode_free_frame_priv(AVRefStructOpaque hwctx, void *data) +{ + AVHWDeviceContext *device_ctx = hwctx.nc; + AVVAAPIDeviceContext *vactx = device_ctx->hwctx; + + vaapi_decode_free_buffers(device_ctx, vactx->display, data); +} + static const struct { uint32_t fourcc; enum AVPixelFormat pix_fmt; diff --git a/libavcodec/vaapi_decode.h b/libavcodec/vaapi_decode.h index 7813473c986..2c696c78910 100644 --- a/libavcodec/vaapi_decode.h +++ b/libavcodec/vaapi_decode.h @@ -24,6 +24,7 @@ #include "libavutil/frame.h" #include "libavutil/hwcontext.h" #include "libavutil/hwcontext_vaapi.h" +#include "libavutil/refstruct.h" #include "avcodec.h" @@ -79,6 +80,7 @@ int ff_vaapi_decode_issue(AVCodecContext *avctx, VAAPIDecodePicture *pic); int ff_vaapi_decode_cancel(AVCodecContext *avctx, VAAPIDecodePicture *pic); +void ff_vaapi_decode_free_frame_priv(AVRefStructOpaque hwctx, void *data); int ff_vaapi_decode_init(AVCodecContext *avctx); int ff_vaapi_decode_uninit(AVCodecContext *avctx); diff --git a/libavcodec/vaapi_h264.c b/libavcodec/vaapi_h264.c index 7f00da09fc1..32ecc97c662 100644 --- a/libavcodec/vaapi_h264.c +++ b/libavcodec/vaapi_h264.c @@ -403,6 +403,7 @@ const FFHWAccel ff_h264_vaapi_hwaccel = { .end_frame = &vaapi_h264_end_frame, .decode_slice = &vaapi_h264_decode_slice, .frame_priv_data_size = sizeof(VAAPIDecodePicture), + .free_frame_priv = &ff_vaapi_decode_free_frame_priv, .init = &ff_vaapi_decode_init, .uninit = &ff_vaapi_decode_uninit, .frame_params = &ff_vaapi_common_frame_params, diff --git a/libavcodec/vaapi_hevc.c b/libavcodec/vaapi_hevc.c index 3d0f6632fd3..4a0202570b1 100644 --- a/libavcodec/vaapi_hevc.c +++ b/libavcodec/vaapi_hevc.c @@ -665,6 +665,7 @@ const FFHWAccel ff_hevc_vaapi_hwaccel = { .end_frame = vaapi_hevc_end_frame, .decode_slice = vaapi_hevc_decode_slice, .frame_priv_data_size = sizeof(VAAPIDecodePictureHEVC), + .free_frame_priv = ff_vaapi_decode_free_frame_priv, .init = ff_vaapi_decode_init, .uninit = ff_vaapi_decode_uninit, .frame_params = ff_vaapi_common_frame_params, diff --git a/libavcodec/vaapi_mjpeg.c b/libavcodec/vaapi_mjpeg.c index c8b59b8afaa..bc05e336a62 100644 --- a/libavcodec/vaapi_mjpeg.c +++ b/libavcodec/vaapi_mjpeg.c @@ -152,6 +152,7 @@ const FFHWAccel ff_mjpeg_vaapi_hwaccel = { .end_frame = &vaapi_mjpeg_end_frame, .decode_slice = &vaapi_mjpeg_decode_slice, .frame_priv_data_size = sizeof(VAAPIDecodePicture), + .free_frame_priv = &ff_vaapi_decode_free_frame_priv, .init = &ff_vaapi_decode_init, .uninit = &ff_vaapi_decode_uninit, .frame_params = &ff_vaapi_common_frame_params, diff --git a/libavcodec/vaapi_mpeg2.c b/libavcodec/vaapi_mpeg2.c index 07bf35c532f..b2a4bc40942 100644 --- a/libavcodec/vaapi_mpeg2.c +++ b/libavcodec/vaapi_mpeg2.c @@ -185,6 +185,7 @@ const FFHWAccel ff_mpeg2_vaapi_hwaccel = { .end_frame = &vaapi_mpeg2_end_frame, .decode_slice = &vaapi_mpeg2_decode_slice, .frame_priv_data_size = sizeof(VAAPIDecodePicture), + .free_frame_priv = &ff_vaapi_decode_free_frame_priv, .init = &ff_vaapi_decode_init, .uninit = &ff_vaapi_decode_uninit, .frame_params = &ff_vaapi_common_frame_params, diff --git a/libavcodec/vaapi_mpeg4.c b/libavcodec/vaapi_mpeg4.c index d936a290cd7..81f0f357ad0 100644 --- a/libavcodec/vaapi_mpeg4.c +++ b/libavcodec/vaapi_mpeg4.c @@ -192,6 +192,7 @@ const FFHWAccel ff_mpeg4_vaapi_hwaccel = { .end_frame = &vaapi_mpeg4_end_frame, .decode_slice = &vaapi_mpeg4_decode_slice, .frame_priv_data_size = sizeof(VAAPIDecodePicture), + .free_frame_priv = &ff_vaapi_decode_free_frame_priv, .init = &ff_vaapi_decode_init, .uninit = &ff_vaapi_decode_uninit, .frame_params = &ff_vaapi_common_frame_params, @@ -210,6 +211,7 @@ const FFHWAccel ff_h263_vaapi_hwaccel = { .end_frame = &vaapi_mpeg4_end_frame, .decode_slice = &vaapi_mpeg4_decode_slice, .frame_priv_data_size = sizeof(VAAPIDecodePicture), + .free_frame_priv = &ff_vaapi_decode_free_frame_priv, .init = &ff_vaapi_decode_init, .uninit = &ff_vaapi_decode_uninit, .frame_params = &ff_vaapi_common_frame_params, diff --git a/libavcodec/vaapi_vc1.c b/libavcodec/vaapi_vc1.c index 42171918e81..7bb2d270555 100644 --- a/libavcodec/vaapi_vc1.c +++ b/libavcodec/vaapi_vc1.c @@ -516,6 +516,7 @@ const FFHWAccel ff_wmv3_vaapi_hwaccel = { .end_frame = &vaapi_vc1_end_frame, .decode_slice = &vaapi_vc1_decode_slice, .frame_priv_data_size = sizeof(VAAPIDecodePicture), + .free_frame_priv = &ff_vaapi_decode_free_frame_priv, .init = &ff_vaapi_decode_init, .uninit = &ff_vaapi_decode_uninit, .frame_params = &ff_vaapi_common_frame_params, @@ -533,6 +534,7 @@ const FFHWAccel ff_vc1_vaapi_hwaccel = { .end_frame = &vaapi_vc1_end_frame, .decode_slice = &vaapi_vc1_decode_slice, .frame_priv_data_size = sizeof(VAAPIDecodePicture), + .free_frame_priv = &ff_vaapi_decode_free_frame_priv, .init = &ff_vaapi_decode_init, .uninit = &ff_vaapi_decode_uninit, .frame_params = &ff_vaapi_common_frame_params, diff --git a/libavcodec/vaapi_vp8.c b/libavcodec/vaapi_vp8.c index b9b1f2aa51e..ec226d53868 100644 --- a/libavcodec/vaapi_vp8.c +++ b/libavcodec/vaapi_vp8.c @@ -230,6 +230,7 @@ const FFHWAccel ff_vp8_vaapi_hwaccel = { .end_frame = &vaapi_vp8_end_frame, .decode_slice = &vaapi_vp8_decode_slice, .frame_priv_data_size = sizeof(VAAPIDecodePicture), + .free_frame_priv = &ff_vaapi_decode_free_frame_priv, .init = &ff_vaapi_decode_init, .uninit = &ff_vaapi_decode_uninit, .frame_params = &ff_vaapi_common_frame_params, diff --git a/libavcodec/vaapi_vp9.c b/libavcodec/vaapi_vp9.c index 7d57f340e0e..f113de96efa 100644 --- a/libavcodec/vaapi_vp9.c +++ b/libavcodec/vaapi_vp9.c @@ -178,6 +178,7 @@ const FFHWAccel ff_vp9_vaapi_hwaccel = { .end_frame = vaapi_vp9_end_frame, .decode_slice = vaapi_vp9_decode_slice, .frame_priv_data_size = sizeof(VAAPIDecodePicture), + .free_frame_priv = ff_vaapi_decode_free_frame_priv, .init = ff_vaapi_decode_init, .uninit = ff_vaapi_decode_uninit, .frame_params = ff_vaapi_common_frame_params, diff --git a/libavcodec/vaapi_vvc.c b/libavcodec/vaapi_vvc.c index f5ee07ee875..175aaba5b19 100644 --- a/libavcodec/vaapi_vvc.c +++ b/libavcodec/vaapi_vvc.c @@ -650,6 +650,7 @@ const FFHWAccel ff_vvc_vaapi_hwaccel = { .end_frame = &vaapi_vvc_end_frame, .decode_slice = &vaapi_vvc_decode_slice, .frame_priv_data_size = sizeof(VAAPIDecodePictureVVC), + .free_frame_priv = &ff_vaapi_decode_free_frame_priv, .init = &ff_vaapi_decode_init, .uninit = &ff_vaapi_decode_uninit, .frame_params = &ff_vaapi_common_frame_params, -- 2.54.0 >From 15358fe694565be50457e1319f69c8aa921c1b2a Mon Sep 17 00:00:00 2001 From: Michael Niedermayer <[email protected]> Date: Sat, 17 Oct 2026 03:35:43 +0200 Subject: [PATCH 8/8] avcodec/cuviddec: keep the decoder and CUarray surfaces alive while zero-copy frames use them Fixes: p6PbEwU20LjP Fixes: AISLE-2026-0150-00017 Fixes: use after free Use after free Replicated through UnModified FFmpeg Found-by: Joshua Rogers (AISLE) The report suggested moving the old decoder and surfaces into a cleanup object referenced by the outstanding frames, as the close path does. --- libavcodec/cuviddec.c | 119 +++++++++++++++++------------------------- 1 file changed, 47 insertions(+), 72 deletions(-) diff --git a/libavcodec/cuviddec.c b/libavcodec/cuviddec.c index 14744cf0679..c60a5eb9a51 100644 --- a/libavcodec/cuviddec.c +++ b/libavcodec/cuviddec.c @@ -266,6 +266,34 @@ static void cuvid_decoder_cleanup_free(void *opaque, uint8_t *data) #endif +static void cuvid_destroy_decoder(AVCodecContext *avctx) +{ + CuvidContext *ctx = avctx->priv_data; + +#ifdef NVDEC_HAVE_OPAQUE_OUTPUT_SUPPORT + if (ctx->decoder_cleanup) { + ctx->decoder_cleanup->cudecoder = ctx->cudecoder; + ctx->decoder_cleanup->cuarray_surfaces = ctx->cuarray_surfaces; + ctx->decoder_cleanup->cuarray_num_surfaces = ctx->cuarray_num_surfaces; + ctx->decoder_cleanup = NULL; + } else { + if (ctx->cudecoder) + CHECK_CU(ctx->cvdl->cuvidDestroyDecoder(ctx->cudecoder)); + for (int i = 0; i < ctx->cuarray_num_surfaces; i++) + CHECK_CU(ctx->cudl->cuArrayDestroy(ctx->cuarray_surfaces[i])); + av_freep(&ctx->cuarray_surfaces); + } + ctx->cuarray_surfaces = NULL; + ctx->cuarray_num_surfaces = 0; + av_buffer_unref(&ctx->surface_in_use_ref); + ctx->surface_in_use = NULL; +#else + if (ctx->cudecoder) + CHECK_CU(ctx->cvdl->cuvidDestroyDecoder(ctx->cudecoder)); +#endif + ctx->cudecoder = NULL; +} + static int CUDAAPI cuvid_handle_video_sequence(void *opaque, CUVIDEOFORMAT* format) { AVCodecContext *avctx = opaque; @@ -479,10 +507,7 @@ static int CUDAAPI cuvid_handle_video_sequence(void *opaque, CUVIDEOFORMAT* form if (ctx->cudecoder) { av_log(avctx, AV_LOG_TRACE, "Re-initializing decoder\n"); - ctx->internal_error = CHECK_CU(ctx->cvdl->cuvidDestroyDecoder(ctx->cudecoder)); - if (ctx->internal_error < 0) - return 0; - ctx->cudecoder = NULL; + cuvid_destroy_decoder(avctx); } if (hwframe_ctx->pool && ( @@ -607,27 +632,13 @@ static int CUDAAPI cuvid_handle_video_sequence(void *opaque, CUVIDEOFORMAT* form ff_nvdec_fill_cuarray_desc(&arr_desc, avctx, cuinfo.OutputFormat); - if (ctx->cuarray_surfaces) { - if (ctx->decoder_cleanup) { - ctx->decoder_cleanup->cuarray_surfaces = NULL; - ctx->decoder_cleanup->cuarray_num_surfaces = 0; - ctx->decoder_cleanup->cudecoder = NULL; - ctx->decoder_cleanup = NULL; - } - for (i = 0; i < ctx->cuarray_num_surfaces; i++) - ctx->cudl->cuArrayDestroy(ctx->cuarray_surfaces[i]); - av_freep(&ctx->cuarray_surfaces); - av_buffer_unref(&ctx->surface_in_use_ref); - ctx->surface_in_use = NULL; - } - ctx->cuarray_num_surfaces = ctx->nb_surfaces; - ctx->cuarray_surfaces = av_calloc(ctx->cuarray_num_surfaces, sizeof(CUarray)); + ctx->cuarray_surfaces = av_calloc(ctx->nb_surfaces, sizeof(CUarray)); if (!ctx->cuarray_surfaces) { ctx->internal_error = AVERROR(ENOMEM); return 0; } { - atomic_int *flags = av_calloc(ctx->cuarray_num_surfaces, sizeof(*flags)); + atomic_int *flags = av_calloc(ctx->nb_surfaces, sizeof(*flags)); CuvidDecoderCleanup *cleanup = NULL; if (!flags) { ctx->internal_error = AVERROR(ENOMEM); @@ -635,40 +646,40 @@ static int CUDAAPI cuvid_handle_video_sequence(void *opaque, CUVIDEOFORMAT* form } if (ctx->zero_copy) { cleanup = av_mallocz(sizeof(*cleanup)); - if (!cleanup) { - av_free((void *)flags); + if (cleanup) { + cleanup->hwdevice = av_buffer_ref(ctx->hwdevice); + if (cleanup->hwdevice) + cuvid_load_functions(&cleanup->cvdl, avctx); + } + if (!cleanup || !cleanup->cvdl) { + cuvid_decoder_cleanup_free(cleanup, (uint8_t *)flags); ctx->internal_error = AVERROR(ENOMEM); return 0; } - ctx->decoder_cleanup = cleanup; } ctx->surface_in_use_ref = av_buffer_create( - (uint8_t *)flags, ctx->cuarray_num_surfaces * sizeof(*flags), + (uint8_t *)flags, ctx->nb_surfaces * sizeof(*flags), cleanup ? cuvid_decoder_cleanup_free : NULL, cleanup, 0); if (!ctx->surface_in_use_ref) { - av_free((void *)flags); - av_free(cleanup); - ctx->decoder_cleanup = NULL; + cuvid_decoder_cleanup_free(cleanup, (uint8_t *)flags); ctx->internal_error = AVERROR(ENOMEM); return 0; } + ctx->decoder_cleanup = cleanup; ctx->surface_in_use = flags; } - for (i = 0; i < ctx->cuarray_num_surfaces; i++) { + for (i = 0; i < ctx->nb_surfaces; i++) { ctx->internal_error = CHECK_CU(ctx->cudl->cuArray3DCreate( &ctx->cuarray_surfaces[i], &arr_desc)); if (ctx->internal_error < 0) { - for (int j = 0; j < i; j++) - ctx->cudl->cuArrayDestroy(ctx->cuarray_surfaces[j]); - av_freep(&ctx->cuarray_surfaces); - av_buffer_unref(&ctx->surface_in_use_ref); - ctx->surface_in_use = NULL; - ctx->cuarray_num_surfaces = 0; + ctx->cuarray_num_surfaces = i; + cuvid_destroy_decoder(avctx); return 0; } } + ctx->cuarray_num_surfaces = ctx->nb_surfaces; { AVHWDeviceContext *dev_ctx = @@ -695,12 +706,6 @@ static int CUDAAPI cuvid_handle_video_sequence(void *opaque, CUVIDEOFORMAT* form CHECK_CU(ctx->cudl->cuCtxPopCurrent(&dummy)); if (ctx->internal_error < 0) return 0; - - if (ctx->decoder_cleanup) { - ctx->decoder_cleanup->cudecoder = ctx->cudecoder; - ctx->decoder_cleanup->cuarray_surfaces = ctx->cuarray_surfaces; - ctx->decoder_cleanup->cuarray_num_surfaces = ctx->cuarray_num_surfaces; - } } #endif @@ -1252,34 +1257,7 @@ static av_cold int cuvid_decode_end(AVCodecContext *avctx) if (ctx->cuparser) ctx->cvdl->cuvidDestroyVideoParser(ctx->cuparser); -#ifdef NVDEC_HAVE_OPAQUE_OUTPUT_SUPPORT - if (ctx->decoder_cleanup) { - ctx->decoder_cleanup->hwdevice = av_buffer_ref(ctx->hwdevice); - ctx->decoder_cleanup->cvdl = ctx->cvdl; - ctx->cvdl = NULL; - ctx->cudecoder = NULL; - ctx->cuarray_surfaces = NULL; - ctx->cuarray_num_surfaces = 0; - av_buffer_unref(&ctx->surface_in_use_ref); - ctx->surface_in_use = NULL; - ctx->decoder_cleanup = NULL; - } else -#endif - { - if (ctx->cudecoder) - ctx->cvdl->cuvidDestroyDecoder(ctx->cudecoder); - -#ifdef NVDEC_HAVE_OPAQUE_OUTPUT_SUPPORT - if (ctx->cuarray_surfaces) { - for (int i = 0; i < ctx->cuarray_num_surfaces; i++) - ctx->cudl->cuArrayDestroy(ctx->cuarray_surfaces[i]); - av_freep(&ctx->cuarray_surfaces); - av_buffer_unref(&ctx->surface_in_use_ref); - ctx->surface_in_use = NULL; - ctx->cuarray_num_surfaces = 0; - } -#endif - } + cuvid_destroy_decoder(avctx); ctx->cudl->cuCtxPopCurrent(&dummy); } @@ -1730,10 +1708,7 @@ static void cuvid_flush(AVCodecContext *avctx) av_fifo_reset2(ctx->frame_queue); - if (ctx->cudecoder) { - ctx->cvdl->cuvidDestroyDecoder(ctx->cudecoder); - ctx->cudecoder = NULL; - } + cuvid_destroy_decoder(avctx); if (ctx->cuparser) { ctx->cvdl->cuvidDestroyVideoParser(ctx->cuparser); -- 2.54.0 _______________________________________________ ffmpeg-devel mailing list -- [email protected] To unsubscribe send an email to [email protected]
