PR #25506 opened by michaelni URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/25506 Patch URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/25506.patch
avfilter/vf_bm3d: bound the search position buffer by the frame size Fixes: heap buffer overflow (write) Fixes: fH770hlenwcI/testcase/repro.sh Fixes: fH770hlenwcI Found-by: Michael Niedermayer <[email protected]> using Depthfirst Out of array write Replicated through UnModified FFmpeg with ASAN >From 5ef156861274517b5d91f0f4b78692436e1ca803 Mon Sep 17 00:00:00 2001 From: Michael Niedermayer <[email protected]> Date: Mon, 26 Oct 2026 16:01:40 +0100 Subject: [PATCH 1/3] avfilter/vf_bm3d: bound the search position buffer by the frame size Fixes: heap buffer overflow (write) Fixes: fH770hlenwcI/testcase/repro.sh Fixes: fH770hlenwcI Found-by: Michael Niedermayer <[email protected]> using Depthfirst Out of array write Replicated through UnModified FFmpeg with ASAN --- libavfilter/vf_bm3d.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/libavfilter/vf_bm3d.c b/libavfilter/vf_bm3d.c index b278d166810..409c7259562 100644 --- a/libavfilter/vf_bm3d.c +++ b/libavfilter/vf_bm3d.c @@ -757,8 +757,6 @@ static int filter_frame(AVFilterContext *ctx, AVFrame **out, AVFrame *in, AVFram return 0; } -#define SQR(x) ((x) * (x)) - static int config_input(AVFilterLink *inlink) { const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(inlink->format); @@ -833,7 +831,10 @@ static int config_input(AVFilterLink *inlink) return AVERROR(ENOMEM); } - sc->search_positions = av_calloc(SQR(2 * s->bm_range / s->bm_step + 1), sizeof(*sc->search_positions)); + const int nb_positions_x = FFMIN(2LL * s->bm_range / s->bm_step, inlink->w / s->bm_step) + 1; + const int nb_positions_y = FFMIN(2LL * s->bm_range / s->bm_step, inlink->h / s->bm_step) + 1; + + sc->search_positions = av_calloc(nb_positions_x * (size_t)nb_positions_y, sizeof(*sc->search_positions)); if (!sc->search_positions) return AVERROR(ENOMEM); } -- 2.54.0 >From c67ed80629e738ffb178f575e282067779ecd9af Mon Sep 17 00:00:00 2001 From: Michael Niedermayer <[email protected]> Date: Mon, 26 Oct 2026 16:02:26 +0100 Subject: [PATCH 2/3] avfilter/vf_bm3d: compute MSE2SSE in double Fixes: signed integer overflow Found during triage/review of the security report fH770hlenwcI Signed integer overflow Replicated through UnModified FFmpeg with UBSAN --- libavfilter/vf_bm3d.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libavfilter/vf_bm3d.c b/libavfilter/vf_bm3d.c index 409c7259562..f71845cd60d 100644 --- a/libavfilter/vf_bm3d.c +++ b/libavfilter/vf_bm3d.c @@ -273,7 +273,7 @@ static void do_block_matching_multi(BM3DContext *s, const uint8_t *src, int src_ int r_y, int r_x, int plane, int jobnr) { SliceContext *sc = &s->slices[jobnr]; - double MSE2SSE = s->group_size * s->block_size * s->block_size * src_range * src_range / (double)(s->max * s->max); + double MSE2SSE = (double)s->group_size * s->block_size * s->block_size * src_range * src_range / ((double)s->max * s->max); double distMul = 1. / MSE2SSE; double th_sse = th_mse * MSE2SSE; int index = sc->nb_match_blocks; -- 2.54.0 >From ac310b843b486c238296d7bd1f671ecaffa7be92 Mon Sep 17 00:00:00 2001 From: Michael Niedermayer <[email protected]> Date: Mon, 26 Oct 2026 16:02:53 +0100 Subject: [PATCH 3/3] avfilter/vf_bm3d: clamp the search range to the plane size Fixes: signed integer overflow Found during triage/review of the security report fH770hlenwcI Signed integer overflow Replicated through UnModified FFmpeg with UBSAN --- libavfilter/vf_bm3d.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libavfilter/vf_bm3d.c b/libavfilter/vf_bm3d.c index f71845cd60d..58910e2414c 100644 --- a/libavfilter/vf_bm3d.c +++ b/libavfilter/vf_bm3d.c @@ -314,7 +314,7 @@ static void block_matching_multi(BM3DContext *s, const uint8_t *ref, int ref_lin const int height = s->planeheight[plane]; const int block_size = s->block_size; const int step = s->bm_step; - const int range = s->bm_range / step * step; + const int range = FFMIN(s->bm_range, FFMAX(width, height)) / step * step; int l = search_boundary(0, range, step, 0, y, x); int r = search_boundary(width - block_size, range, step, 0, y, x); int t = search_boundary(0, range, step, 1, y, x); -- 2.54.0 _______________________________________________ ffmpeg-devel mailing list -- [email protected] To unsubscribe send an email to [email protected]
