PR #24657 opened by michaelni URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/24657 Patch URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/24657.patch
Fixes: out of array write Fixes: KsEohQi9YQw8 Found-by: zhang xingxing Suggested-by: zhang xingxing Signed-off-by: Michael Niedermayer <[email protected]> >From d39da6d2b6bcde99e01a8310734b8ba001395a2f Mon Sep 17 00:00:00 2001 From: Michael Niedermayer <[email protected]> Date: Wed, 23 Sep 2026 23:42:03 +0200 Subject: [PATCH 1/2] avfilter/vf_thumbnail_cuda: clamp the 16-bit histogram bin to 255 Fixes: out of array write Fixes: KsEohQi9YQw8 Found-by: zhang xingxing Suggested-by: zhang xingxing Signed-off-by: Michael Niedermayer <[email protected]> --- libavfilter/vf_thumbnail_cuda.cu | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/libavfilter/vf_thumbnail_cuda.cu b/libavfilter/vf_thumbnail_cuda.cu index d4d4d791f6..350f4da6be 100644 --- a/libavfilter/vf_thumbnail_cuda.cu +++ b/libavfilter/vf_thumbnail_cuda.cu @@ -56,8 +56,8 @@ __global__ void Thumbnail_ushort(cudaTextureObject_t ushort_tex, if (y < src_height && x < src_width) { - unsigned short pixel = (tex2D<unsigned short>(ushort_tex, x, y) + 128) >> 8; - atomicAdd(&histogram[pixel], 1); + int pixel = (tex2D<unsigned short>(ushort_tex, x, y) + 128) >> 8; + atomicAdd(&histogram[min(pixel, 255)], 1); } } @@ -70,8 +70,8 @@ __global__ void Thumbnail_ushort2(cudaTextureObject_t ushort2_tex, if (y < src_height && x < src_width) { ushort2 pixel = tex2D<ushort2>(ushort2_tex, x, y); - atomicAdd(&histogram[(pixel.x + 128) >> 8], 1); - atomicAdd(&histogram[256 + ((pixel.y + 128) >> 8)], 1); + atomicAdd(&histogram[min((pixel.x + 128) >> 8, 255)], 1); + atomicAdd(&histogram[256 + min((pixel.y + 128) >> 8, 255)], 1); } } -- 2.52.0 >From eb96403123938e37a69ccef294e6716f7457e187 Mon Sep 17 00:00:00 2001 From: Michael Niedermayer <[email protected]> Date: Tue, 22 Sep 2026 00:40:29 +0200 Subject: [PATCH 2/2] avfilter/vf_thumbnail_cuda: use the single channel kernel for planar 16-bit formats Fixes: out of array write on every frame of a planar 16-bit format Fixes: KsEohQi9YQw8 Found during triage/review of the security report Signed-off-by: Michael Niedermayer <[email protected]> --- libavfilter/vf_thumbnail_cuda.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/libavfilter/vf_thumbnail_cuda.c b/libavfilter/vf_thumbnail_cuda.c index 95c46eefdb..9cd1bb042b 100644 --- a/libavfilter/vf_thumbnail_cuda.c +++ b/libavfilter/vf_thumbnail_cuda.c @@ -239,11 +239,11 @@ static int thumbnail(AVFilterContext *ctx, int *histogram, AVFrame *in) case AV_PIX_FMT_YUV444P10MSB: case AV_PIX_FMT_YUV444P12MSB: case AV_PIX_FMT_YUV444P16: - thumbnail_kernel(ctx, s->cu_func_ushort2, 1, + thumbnail_kernel(ctx, s->cu_func_ushort, 1, histogram, in->data[0], in->width, in->height, in->linesize[0], 2); - thumbnail_kernel(ctx, s->cu_func_ushort2, 1, + thumbnail_kernel(ctx, s->cu_func_ushort, 1, histogram + 256, in->data[1], in->width, in->height, in->linesize[1], 2); - thumbnail_kernel(ctx, s->cu_func_ushort2, 1, + thumbnail_kernel(ctx, s->cu_func_ushort, 1, histogram + 512, in->data[2], in->width, in->height, in->linesize[2], 2); break; default: -- 2.52.0 _______________________________________________ ffmpeg-devel mailing list -- [email protected] To unsubscribe send an email to [email protected]
