PR #23283 opened by michaelni URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/23283 Patch URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/23283.patch
>From 28cbfc7984952dc192fb353b43b5b35bf58fde5a Mon Sep 17 00:00:00 2001 From: Michael Niedermayer <[email protected]> Date: Sat, 30 May 2026 22:37:49 +0200 Subject: [PATCH 1/2] avcodec/vc2enc_dwt: avoid signed overflow in the 9/7 DWT lifting Fixes: 490488944/clusterfuzz-testcase-minimized-ffmpeg_AV_CODEC_ID_VC2_fuzzer-5310290362433536 Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg Signed-off-by: Michael Niedermayer <[email protected]> --- libavcodec/vc2enc_dwt.c | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/libavcodec/vc2enc_dwt.c b/libavcodec/vc2enc_dwt.c index 809f938e1b..e59755045c 100644 --- a/libavcodec/vc2enc_dwt.c +++ b/libavcodec/vc2enc_dwt.c @@ -56,7 +56,7 @@ static void vc2_subband_dwt_97(VC2TransformContext *t, dwtcoef *data, ptrdiff_t stride, int width, int height) { int x, y; - dwtcoef *datal = data, *synth = t->buffer, *synthl = synth; + uint32_t *datal = (uint32_t *)data, *synth = (uint32_t *)t->buffer, *synthl = synth; const ptrdiff_t synth_width = width << 1; const ptrdiff_t synth_height = height << 1; @@ -75,21 +75,21 @@ static void vc2_subband_dwt_97(VC2TransformContext *t, dwtcoef *data, synthl = synth; for (y = 0; y < synth_height; y++) { /* Lifting stage 2. */ - synthl[1] -= (8*synthl[0] + 9*synthl[2] - synthl[4] + 8) >> 4; + synthl[1] -= (dwtcoef)(8*synthl[0] + 9*synthl[2] - synthl[4] + 8) >> 4; for (x = 1; x < width - 2; x++) - synthl[2*x + 1] -= (9*synthl[2*x] + 9*synthl[2*x + 2] - synthl[2*x + 4] - + synthl[2*x + 1] -= (dwtcoef)(9*synthl[2*x] + 9*synthl[2*x + 2] - synthl[2*x + 4] - synthl[2 * x - 2] + 8) >> 4; - synthl[synth_width - 1] -= (17*synthl[synth_width - 2] - + synthl[synth_width - 1] -= (dwtcoef)(17*synthl[synth_width - 2] - synthl[synth_width - 4] + 8) >> 4; - synthl[synth_width - 3] -= (8*synthl[synth_width - 2] + + synthl[synth_width - 3] -= (dwtcoef)(8*synthl[synth_width - 2] + 9*synthl[synth_width - 4] - synthl[synth_width - 6] + 8) >> 4; /* Lifting stage 1. */ - synthl[0] += (synthl[1] + synthl[1] + 2) >> 2; + synthl[0] += (dwtcoef)(synthl[1] + synthl[1] + 2) >> 2; for (x = 1; x < width - 1; x++) - synthl[2*x] += (synthl[2*x - 1] + synthl[2*x + 1] + 2) >> 2; + synthl[2*x] += (dwtcoef)(synthl[2*x - 1] + synthl[2*x + 1] + 2) >> 2; - synthl[synth_width - 2] += (synthl[synth_width - 3] + + synthl[synth_width - 2] += (dwtcoef)(synthl[synth_width - 3] + synthl[synth_width - 1] + 2) >> 2; synthl += synth_width; } @@ -97,13 +97,13 @@ static void vc2_subband_dwt_97(VC2TransformContext *t, dwtcoef *data, /* Vertical synthesis: Lifting stage 2. */ synthl = synth + synth_width; for (x = 0; x < synth_width; x++) - synthl[x] -= (8*synthl[x - synth_width] + 9*synthl[x + synth_width] - + synthl[x] -= (dwtcoef)(8*synthl[x - synth_width] + 9*synthl[x + synth_width] - synthl[x + 3 * synth_width] + 8) >> 4; synthl = synth + (synth_width << 1); for (y = 1; y < height - 2; y++) { for (x = 0; x < synth_width; x++) - synthl[x + synth_width] -= (9*synthl[x] + + synthl[x + synth_width] -= (dwtcoef)(9*synthl[x] + 9*synthl[x + 2 * synth_width] - synthl[x - 2 * synth_width] - synthl[x + 4 * synth_width] + 8) >> 4; @@ -112,29 +112,29 @@ static void vc2_subband_dwt_97(VC2TransformContext *t, dwtcoef *data, synthl = synth + (synth_height - 1) * synth_width; for (x = 0; x < synth_width; x++) { - synthl[x] -= (17*synthl[x - synth_width] - + synthl[x] -= (dwtcoef)(17*synthl[x - synth_width] - synthl[x - 3*synth_width] + 8) >> 4; - synthl[x - 2*synth_width] -= (9*synthl[x - 3*synth_width] + + synthl[x - 2*synth_width] -= (dwtcoef)(9*synthl[x - 3*synth_width] + 8*synthl[x - 1*synth_width] - synthl[x - 5*synth_width] + 8) >> 4; } /* Vertical synthesis: Lifting stage 1. */ synthl = synth; for (x = 0; x < synth_width; x++) - synthl[x] += (synthl[x + synth_width] + synthl[x + synth_width] + 2) >> 2; + synthl[x] += (dwtcoef)(synthl[x + synth_width] + synthl[x + synth_width] + 2) >> 2; synthl = synth + (synth_width << 1); for (y = 1; y < height - 1; y++) { for (x = 0; x < synth_width; x++) - synthl[x] += (synthl[x - synth_width] + synthl[x + synth_width] + 2) >> 2; + synthl[x] += (dwtcoef)(synthl[x - synth_width] + synthl[x + synth_width] + 2) >> 2; synthl += synth_width << 1; } synthl = synth + (synth_height - 2) * synth_width; for (x = 0; x < synth_width; x++) - synthl[x] += (synthl[x - synth_width] + synthl[x + synth_width] + 2) >> 2; + synthl[x] += (dwtcoef)(synthl[x - synth_width] + synthl[x + synth_width] + 2) >> 2; - deinterleave(data, stride, width, height, synth); + deinterleave(data, stride, width, height, (dwtcoef *)synth); } static void vc2_subband_dwt_53(VC2TransformContext *t, dwtcoef *data, -- 2.52.0 >From 5542376a5ea32131026cdd87267d153bb6082787 Mon Sep 17 00:00:00 2001 From: Michael Niedermayer <[email protected]> Date: Sat, 30 May 2026 22:42:50 +0200 Subject: [PATCH 2/2] avcodec/vc2enc_dwt: avoid signed overflow in the 5/3 and Haar DWT Signed-off-by: Michael Niedermayer <[email protected]> --- libavcodec/vc2enc_dwt.c | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/libavcodec/vc2enc_dwt.c b/libavcodec/vc2enc_dwt.c index e59755045c..5fb2642c92 100644 --- a/libavcodec/vc2enc_dwt.c +++ b/libavcodec/vc2enc_dwt.c @@ -141,7 +141,7 @@ static void vc2_subband_dwt_53(VC2TransformContext *t, dwtcoef *data, ptrdiff_t stride, int width, int height) { int x, y; - dwtcoef *synth = t->buffer, *synthl = synth, *datal = data; + uint32_t *synth = (uint32_t *)t->buffer, *synthl = synth, *datal = (uint32_t *)data; const ptrdiff_t synth_width = width << 1; const ptrdiff_t synth_height = height << 1; @@ -161,16 +161,16 @@ static void vc2_subband_dwt_53(VC2TransformContext *t, dwtcoef *data, for (y = 0; y < synth_height; y++) { /* Lifting stage 2. */ for (x = 0; x < width - 1; x++) - synthl[2 * x + 1] -= (synthl[2 * x] + synthl[2 * x + 2] + 1) >> 1; + synthl[2 * x + 1] -= (dwtcoef)(synthl[2 * x] + synthl[2 * x + 2] + 1) >> 1; - synthl[synth_width - 1] -= (2*synthl[synth_width - 2] + 1) >> 1; + synthl[synth_width - 1] -= (dwtcoef)(2*synthl[synth_width - 2] + 1) >> 1; /* Lifting stage 1. */ - synthl[0] += (2*synthl[1] + 2) >> 2; + synthl[0] += (dwtcoef)(2*synthl[1] + 2) >> 2; for (x = 1; x < width - 1; x++) - synthl[2 * x] += (synthl[2 * x - 1] + synthl[2 * x + 1] + 2) >> 2; + synthl[2 * x] += (dwtcoef)(synthl[2 * x - 1] + synthl[2 * x + 1] + 2) >> 2; - synthl[synth_width - 2] += (synthl[synth_width - 3] + synthl[synth_width - 1] + 2) >> 2; + synthl[synth_width - 2] += (dwtcoef)(synthl[synth_width - 3] + synthl[synth_width - 1] + 2) >> 2; synthl += synth_width; } @@ -178,37 +178,37 @@ static void vc2_subband_dwt_53(VC2TransformContext *t, dwtcoef *data, /* Vertical synthesis: Lifting stage 2. */ synthl = synth + synth_width; for (x = 0; x < synth_width; x++) - synthl[x] -= (synthl[x - synth_width] + synthl[x + synth_width] + 1) >> 1; + synthl[x] -= (dwtcoef)(synthl[x - synth_width] + synthl[x + synth_width] + 1) >> 1; synthl = synth + (synth_width << 1); for (y = 1; y < height - 1; y++) { for (x = 0; x < synth_width; x++) - synthl[x + synth_width] -= (synthl[x] + synthl[x + synth_width * 2] + 1) >> 1; + synthl[x + synth_width] -= (dwtcoef)(synthl[x] + synthl[x + synth_width * 2] + 1) >> 1; synthl += (synth_width << 1); } synthl = synth + (synth_height - 1) * synth_width; for (x = 0; x < synth_width; x++) - synthl[x] -= (2*synthl[x - synth_width] + 1) >> 1; + synthl[x] -= (dwtcoef)(2*synthl[x - synth_width] + 1) >> 1; /* Vertical synthesis: Lifting stage 1. */ synthl = synth; for (x = 0; x < synth_width; x++) - synthl[x] += (2*synthl[synth_width + x] + 2) >> 2; + synthl[x] += (dwtcoef)(2*synthl[synth_width + x] + 2) >> 2; synthl = synth + (synth_width << 1); for (y = 1; y < height - 1; y++) { for (x = 0; x < synth_width; x++) - synthl[x] += (synthl[x + synth_width] + synthl[x - synth_width] + 2) >> 2; + synthl[x] += (dwtcoef)(synthl[x + synth_width] + synthl[x - synth_width] + 2) >> 2; synthl += (synth_width << 1); } synthl = synth + (synth_height - 2)*synth_width; for (x = 0; x < synth_width; x++) - synthl[x] += (synthl[x - synth_width] + synthl[x + synth_width] + 2) >> 2; + synthl[x] += (dwtcoef)(synthl[x - synth_width] + synthl[x + synth_width] + 2) >> 2; - deinterleave(data, stride, width, height, synth); + deinterleave(data, stride, width, height, (dwtcoef *)synth); } static av_always_inline void dwt_haar(VC2TransformContext *t, dwtcoef *data, @@ -216,7 +216,7 @@ static av_always_inline void dwt_haar(VC2TransformContext *t, dwtcoef *data, const int s) { int x, y; - dwtcoef *synth = t->buffer, *synthl = synth, *datal = data; + uint32_t *synth = (uint32_t *)t->buffer, *synthl = synth, *datal = (uint32_t *)data; const ptrdiff_t synth_width = width << 1; const ptrdiff_t synth_height = height << 1; @@ -225,7 +225,7 @@ static av_always_inline void dwt_haar(VC2TransformContext *t, dwtcoef *data, for (x = 0; x < synth_width; x += 2) { synthl[y*synth_width + x + 1] = (datal[y*stride + x + 1] - datal[y*stride + x]) * (1 << s); synthl[y*synth_width + x] = datal[y*stride + x + 0] * (1 << s) + - ((synthl[y*synth_width + x + 1] + 1) >> 1); + ((dwtcoef)(synthl[y*synth_width + x + 1] + 1) >> 1); } } @@ -235,11 +235,11 @@ static av_always_inline void dwt_haar(VC2TransformContext *t, dwtcoef *data, synthl[(y + 1)*synth_width + x] = synthl[(y + 1)*synth_width + x] - synthl[y*synth_width + x]; synthl[y*synth_width + x] = synthl[y*synth_width + x] + - ((synthl[(y + 1)*synth_width + x] + 1) >> 1); + ((dwtcoef)(synthl[(y + 1)*synth_width + x] + 1) >> 1); } } - deinterleave(data, stride, width, height, synth); + deinterleave(data, stride, width, height, (dwtcoef *)synth); } static void vc2_subband_dwt_haar(VC2TransformContext *t, dwtcoef *data, -- 2.52.0 _______________________________________________ ffmpeg-devel mailing list -- [email protected] To unsubscribe send an email to [email protected]
