PR #23780 opened by michaelni URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/23780 Patch URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/23780.patch
Fixes: out of array access Fixes: 8aj_floodfill_dynamic_size.pgm / 8aj_generate_floodfill_dynamic_size_pgm.py Fixes: 3MleMXjGZvu3 Found-by: Adrian Junge (vurlo) <[email protected]> >From 197427d4b954a8ea1650720a87ee547ac10853fb Mon Sep 17 00:00:00 2001 From: Michael Niedermayer <[email protected]> Date: Sat, 11 Jul 2026 16:47:28 +0200 Subject: [PATCH 1/2] avfilter/vf_floodfill: size the point stack for the current frame Fixes: out of array access Fixes: 8aj_floodfill_dynamic_size.pgm / 8aj_generate_floodfill_dynamic_size_pgm.py Fixes: 3MleMXjGZvu3 Found-by: Adrian Junge (vurlo) <[email protected]> --- libavfilter/vf_floodfill.c | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/libavfilter/vf_floodfill.c b/libavfilter/vf_floodfill.c index 6d89963e71..e569d5f586 100644 --- a/libavfilter/vf_floodfill.c +++ b/libavfilter/vf_floodfill.c @@ -41,6 +41,7 @@ typedef struct FloodfillContext { int nb_planes; int back, front; Points *points; + unsigned int points_size; int (*is_same)(const AVFrame *frame, int x, int y, unsigned s0, unsigned s1, unsigned s2, unsigned s3); @@ -271,9 +272,6 @@ static int config_input(AVFilterLink *inlink) } s->front = s->back = 0; - s->points = av_calloc(inlink->w * inlink->h, 4 * sizeof(Points)); - if (!s->points) - return AVERROR(ENOMEM); return 0; } @@ -292,8 +290,23 @@ static int filter_frame(AVFilterLink *link, AVFrame *frame) int s3 = s->s[3]; const int w = frame->width; const int h = frame->height; + size_t nb_points, points_size; int i, ret; + if (w > UINT16_MAX + 1 || h > UINT16_MAX + 1 || + av_size_mult(w, h, &nb_points) < 0 || + av_size_mult(nb_points, 4 * sizeof(*s->points), &points_size) < 0) { + av_frame_free(&frame); + return AVERROR(EINVAL); + } + + av_fast_malloc(&s->points, &s->points_size, points_size); + if (!s->points) { + av_frame_free(&frame); + return AVERROR(ENOMEM); + } + s->front = s->back = 0; + if (is_inside(s->x, s->y, w, h)) { s->pick_pixel(frame, s->x, s->y, &s0, &s1, &s2, &s3); -- 2.52.0 >From 32b0a96f2ce5e47a26c56523dcf0e2a8835ce015 Mon Sep 17 00:00:00 2001 From: Michael Niedermayer <[email protected]> Date: Sun, 12 Jul 2026 03:27:47 +0200 Subject: [PATCH 2/2] avfilter/vf_floodfill: remove unneeded variables Signed-off-by: Michael Niedermayer <[email protected]> --- libavfilter/vf_floodfill.c | 35 ++++++++++++++++------------------- 1 file changed, 16 insertions(+), 19 deletions(-) diff --git a/libavfilter/vf_floodfill.c b/libavfilter/vf_floodfill.c index e569d5f586..9bc72e2a78 100644 --- a/libavfilter/vf_floodfill.c +++ b/libavfilter/vf_floodfill.c @@ -39,7 +39,6 @@ typedef struct FloodfillContext { int d[4]; int nb_planes; - int back, front; Points *points; unsigned int points_size; @@ -271,8 +270,6 @@ static int config_input(AVFilterLink *inlink) } } - s->front = s->back = 0; - return 0; } @@ -292,6 +289,7 @@ static int filter_frame(AVFilterLink *link, AVFrame *frame) const int h = frame->height; size_t nb_points, points_size; int i, ret; + int front = 0; if (w > UINT16_MAX + 1 || h > UINT16_MAX + 1 || av_size_mult(w, h, &nb_points) < 0 || @@ -305,7 +303,6 @@ static int filter_frame(AVFilterLink *link, AVFrame *frame) av_frame_free(&frame); return AVERROR(ENOMEM); } - s->front = s->back = 0; if (is_inside(s->x, s->y, w, h)) { s->pick_pixel(frame, s->x, s->y, &s0, &s1, &s2, &s3); @@ -323,9 +320,9 @@ static int filter_frame(AVFilterLink *link, AVFrame *frame) goto end; if (s->is_same(frame, s->x, s->y, s0, s1, s2, s3)) { - s->points[s->front].x = s->x; - s->points[s->front].y = s->y; - s->front++; + s->points[front].x = s->x; + s->points[front].y = s->y; + front++; } if (ret = ff_inlink_make_frame_writable(link, &frame)) { @@ -333,34 +330,34 @@ static int filter_frame(AVFilterLink *link, AVFrame *frame) return ret; } - while (s->front > s->back) { + while (front > 0) { int x, y; - s->front--; - x = s->points[s->front].x; - y = s->points[s->front].y; + front--; + x = s->points[front].x; + y = s->points[front].y; if (s->is_same(frame, x, y, s0, s1, s2, s3)) { s->set_pixel(frame, x, y, d0, d1, d2, d3); if (is_inside(x + 1, y, w, h)) { - s->points[s->front] .x = x + 1; - s->points[s->front++].y = y; + s->points[front] .x = x + 1; + s->points[front++].y = y; } if (is_inside(x - 1, y, w, h)) { - s->points[s->front] .x = x - 1; - s->points[s->front++].y = y; + s->points[front] .x = x - 1; + s->points[front++].y = y; } if (is_inside(x, y + 1, w, h)) { - s->points[s->front] .x = x; - s->points[s->front++].y = y + 1; + s->points[front] .x = x; + s->points[front++].y = y + 1; } if (is_inside(x, y - 1, w, h)) { - s->points[s->front] .x = x; - s->points[s->front++].y = y - 1; + s->points[front] .x = x; + s->points[front++].y = y - 1; } } } -- 2.52.0 _______________________________________________ ffmpeg-devel mailing list -- [email protected] To unsubscribe send an email to [email protected]
