PR #24471 opened by michaelni URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/24471 Patch URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/24471.patch
Fixes: out of array access Fixes: index -2147483648 out of bounds for type 'float[35768]' Fixes: shift exponent 37 is too large for 32-bit type 'int' Fixes: index -2147483648 out of bounds for type 'float[35768]' Fixes: -nan is outside the range of representable values of type 'int' Fixes: index 35768 out of bounds for type 'float[35768]' Fixes: 9UQo8TUCJdYw Found-by: Zheng Yu <[email protected]> >From e7030fc7f2b0e3be4d3742c5c58598b591b50ee2 Mon Sep 17 00:00:00 2001 From: Michael Niedermayer <[email protected]> Date: Mon, 7 Sep 2026 00:36:04 +0200 Subject: [PATCH 1/6] avcodec/nellymoserenc: keep the trellis index below OPT_SIZE Fixes: out of array access Fixes: 9UQo8TUCJdYw Found-by: Zheng Yu <[email protected]> --- libavcodec/nellymoserenc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libavcodec/nellymoserenc.c b/libavcodec/nellymoserenc.c index 54b598dbb1..408193ad07 100644 --- a/libavcodec/nellymoserenc.c +++ b/libavcodec/nellymoserenc.c @@ -259,7 +259,7 @@ static void get_exponent_dynamic(NellyMoserEncodeContext *s, float *cand, int *i power_candidate = cand[band]; for (q = 1000; !c && q < OPT_SIZE; q <<= 2) { idx_min = FFMAX(0, cand[band] - q); - idx_max = FFMIN(OPT_SIZE, cand[band - 1] + q); + idx_max = FFMIN(OPT_SIZE - 1, cand[band - 1] + q); for (i = FFMAX(0, cand[band - 1] - q); i < FFMIN(OPT_SIZE, cand[band - 1] + q); i++) { if ( isinf(opt[band - 1][i]) ) continue; -- 2.52.0 >From c22f1e260f894ec99e9bd82ffad0fadbaa89e20c Mon Sep 17 00:00:00 2001 From: Michael Niedermayer <[email protected]> Date: Mon, 7 Sep 2026 00:37:01 +0200 Subject: [PATCH 2/6] avcodec/nellymoserenc: initialize the trellis table row by row Fixes: index 35768 out of bounds for type 'float[35768]' Fixes: 9UQo8TUCJdYw Found-by: Zheng Yu <[email protected]> --- libavcodec/nellymoserenc.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/libavcodec/nellymoserenc.c b/libavcodec/nellymoserenc.c index 408193ad07..af516eb82e 100644 --- a/libavcodec/nellymoserenc.c +++ b/libavcodec/nellymoserenc.c @@ -243,9 +243,9 @@ static void get_exponent_dynamic(NellyMoserEncodeContext *s, float *cand, int *i float (*opt )[OPT_SIZE] = s->opt ; uint8_t(*path)[OPT_SIZE] = s->path; - for (i = 0; i < NELLY_BANDS * OPT_SIZE; i++) { - opt[0][i] = INFINITY; - } + for (band = 0; band < NELLY_BANDS; band++) + for (i = 0; i < OPT_SIZE; i++) + opt[band][i] = INFINITY; for (i = 0; i < 64; i++) { opt[0][ff_nelly_init_table[i]] = distance(cand[0], ff_nelly_init_table[i], 0); -- 2.52.0 >From 6973015a1c25db11ddfe588a577f47e81c505dcc Mon Sep 17 00:00:00 2001 From: Michael Niedermayer <[email protected]> Date: Sun, 13 Sep 2026 16:47:05 +0200 Subject: [PATCH 3/6] avcodec/nellymoserenc: widen the trellis search until it covers the table Fixes: out of array read Fixes: 9UQo8TUCJdYw --- libavcodec/nellymoserenc.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libavcodec/nellymoserenc.c b/libavcodec/nellymoserenc.c index af516eb82e..d797e3dcf0 100644 --- a/libavcodec/nellymoserenc.c +++ b/libavcodec/nellymoserenc.c @@ -257,7 +257,7 @@ static void get_exponent_dynamic(NellyMoserEncodeContext *s, float *cand, int *i float tmp; int idx_min, idx_max, idx; power_candidate = cand[band]; - for (q = 1000; !c && q < OPT_SIZE; q <<= 2) { + for (q = 1000; !c && q < 2 * OPT_SIZE; q <<= 2) { idx_min = FFMAX(0, cand[band] - q); idx_max = FFMIN(OPT_SIZE - 1, cand[band - 1] + q); for (i = FFMAX(0, cand[band - 1] - q); i < FFMIN(OPT_SIZE, cand[band - 1] + q); i++) { @@ -278,7 +278,7 @@ static void get_exponent_dynamic(NellyMoserEncodeContext *s, float *cand, int *i } } } - av_assert1(c); //FIXME + av_assert1(c); } best_val = INFINITY; -- 2.52.0 >From c5f5175e2034666e0d838cdddb984f11ada1ca35 Mon Sep 17 00:00:00 2001 From: Michael Niedermayer <[email protected]> Date: Sun, 13 Sep 2026 18:38:28 +0200 Subject: [PATCH 4/6] avcodec/nellymoserenc: check the band exponents for finiteness Fixes: index -2147483648 out of bounds for type 'float[35768]' Fixes: -nan is outside the range of representable values of type 'int' Fixes: 9UQo8TUCJdYw --- libavcodec/nellymoserenc.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/libavcodec/nellymoserenc.c b/libavcodec/nellymoserenc.c index d797e3dcf0..98bf563756 100644 --- a/libavcodec/nellymoserenc.c +++ b/libavcodec/nellymoserenc.c @@ -304,7 +304,7 @@ static void get_exponent_dynamic(NellyMoserEncodeContext *s, float *cand, int *i * @param output output buffer * @param output_size size of output buffer */ -static void encode_block(NellyMoserEncodeContext *s, unsigned char *output, int output_size) +static int encode_block(NellyMoserEncodeContext *s, unsigned char *output, int output_size) { PutBitContext pb; int i, j, band, block, best_idx, power_idx = 0; @@ -326,6 +326,10 @@ static void encode_block(NellyMoserEncodeContext *s, unsigned char *output, int } cand[band] = log2(FFMAX(1.0, coeff_sum / (ff_nelly_band_sizes_table[band] << 7))) * 1024.0; + if (!isfinite(cand[band])) { + av_log(s->avctx, AV_LOG_ERROR, "Input contains NaN/+-Inf\n"); + return AVERROR(EINVAL); + } } if (s->avctx->trellis) { @@ -376,6 +380,7 @@ static void encode_block(NellyMoserEncodeContext *s, unsigned char *output, int flush_put_bits(&pb); memset(put_bits_ptr(&pb), 0, output + output_size - put_bits_ptr(&pb)); + return 0; } static int encode_frame(AVCodecContext *avctx, AVPacket *avpkt, @@ -406,7 +411,8 @@ static int encode_frame(AVCodecContext *avctx, AVPacket *avpkt, if ((ret = ff_get_encode_buffer(avctx, avpkt, NELLY_BLOCK_LEN, 0)) < 0) return ret; - encode_block(s, avpkt->data, avpkt->size); + if ((ret = encode_block(s, avpkt->data, avpkt->size)) < 0) + return ret; /* Get the next frame pts/duration */ ret = ff_af_queue_remove(&s->afq, avctx->frame_size, avpkt); -- 2.52.0 >From e40b5a9d432f90ba003a15724ad87ff9f24e90fb Mon Sep 17 00:00:00 2001 From: Michael Niedermayer <[email protected]> Date: Sun, 13 Sep 2026 20:55:25 +0200 Subject: [PATCH 5/6] avcodec/nellymoserenc: refuse exponents that overflow the scale factor shift Fixes: shift exponent 37 is too large for 32-bit type 'int' Fixes: 9UQo8TUCJdYw --- libavcodec/nellymoserenc.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libavcodec/nellymoserenc.c b/libavcodec/nellymoserenc.c index 98bf563756..5de624b7dc 100644 --- a/libavcodec/nellymoserenc.c +++ b/libavcodec/nellymoserenc.c @@ -347,6 +347,8 @@ static int encode_block(NellyMoserEncodeContext *s, unsigned char *output, int o power_idx = ff_nelly_init_table[idx_table[0]]; put_bits(&pb, 6, idx_table[0]); } + if (power_idx >= (31 - POW_TABLE_OFFSET) << 11) + return AVERROR(EINVAL); power_val = pow_table[power_idx & 0x7FF] / (1 << ((power_idx >> 11) + POW_TABLE_OFFSET)); for (j = 0; j < ff_nelly_band_sizes_table[band]; i++, j++) { s->mdct_out[i] *= power_val; -- 2.52.0 >From 8ee1ec8a9395eefd62ede6d4fa5ce3388f7c22cf Mon Sep 17 00:00:00 2001 From: Michael Niedermayer <[email protected]> Date: Sun, 13 Sep 2026 21:22:03 +0200 Subject: [PATCH 6/6] avcodec/nellymoserenc: handle trellis implementation failure Fixes: index -2147483648 out of bounds for type 'float[35768]' Fixes: out of array read Fixes: 9UQo8TUCJdYw Signed-off-by: Michael Niedermayer <[email protected]> --- libavcodec/nellymoserenc.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/libavcodec/nellymoserenc.c b/libavcodec/nellymoserenc.c index 5de624b7dc..197bb1c1a2 100644 --- a/libavcodec/nellymoserenc.c +++ b/libavcodec/nellymoserenc.c @@ -235,7 +235,7 @@ static inline float distance(float x, float y, int band) return tmp * tmp; } -static void get_exponent_dynamic(NellyMoserEncodeContext *s, float *cand, int *idx_table) +static int get_exponent_dynamic(NellyMoserEncodeContext *s, float *cand, int *idx_table) { int i, j, band, best_idx; float power_candidate, best_val; @@ -278,7 +278,6 @@ static void get_exponent_dynamic(NellyMoserEncodeContext *s, float *cand, int *i } } } - av_assert1(c); } best_val = INFINITY; @@ -290,12 +289,15 @@ static void get_exponent_dynamic(NellyMoserEncodeContext *s, float *cand, int *i best_idx = i; } } + if (best_idx < 0) + return AVERROR(EINVAL); for (band = NELLY_BANDS - 1; band >= 0; band--) { idx_table[band] = path[band][best_idx]; if (band) { best_idx -= ff_nelly_delta_table[path[band][best_idx]]; } } + return 0; } /** @@ -333,7 +335,9 @@ static int encode_block(NellyMoserEncodeContext *s, unsigned char *output, int o } if (s->avctx->trellis) { - get_exponent_dynamic(s, cand, idx_table); + int ret = get_exponent_dynamic(s, cand, idx_table); + if (ret < 0) + return ret; } else { get_exponent_greedy(s, cand, idx_table); } -- 2.52.0 _______________________________________________ ffmpeg-devel mailing list -- [email protected] To unsubscribe send an email to [email protected]
