Fixes: OOM
Fixes: 3541/clusterfuzz-testcase-minimized-6469958596820992
Adds support for decoding codeblock data larger than 8kb
Reduces decoder memory consumption
Found-by: continuous fuzzing process
https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg
Signed-off-by: Michael Niedermayer
---
libavcodec/j2kenc.c | 8 ++--
libavcodec/jpeg2000.c| 12 ++--
libavcodec/jpeg2000.h| 4 ++--
libavcodec/jpeg2000dec.c | 22 +++---
4 files changed, 37 insertions(+), 9 deletions(-)
diff --git a/libavcodec/j2kenc.c b/libavcodec/j2kenc.c
index 2135ece540..362c7f0f49 100644
--- a/libavcodec/j2kenc.c
+++ b/libavcodec/j2kenc.c
@@ -663,7 +663,8 @@ static void encode_cblk(Jpeg2000EncoderContext *s,
Jpeg2000T1Context *t1, Jpeg20
bpno = cblk->nonzerobits - 1;
}
-ff_mqc_initenc(&t1->mqc, cblk->data);
+cblk->data[0] = 0;
+ff_mqc_initenc(&t1->mqc, cblk->data + 1);
for (passno = 0; bpno >= 0; passno++){
nmsedec=0;
@@ -796,7 +797,7 @@ static int encode_packet(Jpeg2000EncoderContext *s,
Jpeg2000ResLevel *rlevel, in
if (cblk->ninclpasses){
if (s->buf_end - s->buf <
cblk->passes[cblk->ninclpasses-1].rate)
return -1;
-bytestream_put_buffer(&s->buf, cblk->data,
cblk->passes[cblk->ninclpasses-1].rate
+bytestream_put_buffer(&s->buf, cblk->data + 1,
cblk->passes[cblk->ninclpasses-1].rate
-
cblk->passes[cblk->ninclpasses-1].flushed_len);
bytestream_put_buffer(&s->buf,
cblk->passes[cblk->ninclpasses-1].flushed,
cblk->passes[cblk->ninclpasses-1].flushed_len);
@@ -937,6 +938,9 @@ static int encode_tile(Jpeg2000EncoderContext *s,
Jpeg2000Tile *tile, int tileno
}
}
}
+prec->cblk[cblkno].data = av_malloc(1 + 8192);
+if (!prec->cblk[cblkno].data)
+return AVERROR(ENOMEM);
encode_cblk(s, &t1, prec->cblk + cblkno, tile, xx1 -
xx0, yy1 - yy0,
bandpos, codsty->nreslevels - reslevelno -
1);
xx0 = xx1;
diff --git a/libavcodec/jpeg2000.c b/libavcodec/jpeg2000.c
index afeb9df27c..8551cf8d6c 100644
--- a/libavcodec/jpeg2000.c
+++ b/libavcodec/jpeg2000.c
@@ -357,7 +357,6 @@ static int init_prec(Jpeg2000Band *band,
comp->reslevel[reslevelno-1].coord[1][0];
}
-cblk->zero = 0;
cblk->lblock= 3;
cblk->length= 0;
memset(cblk->lengthinc, 0, sizeof(cblk->lengthinc));
@@ -598,9 +597,18 @@ void ff_jpeg2000_cleanup(Jpeg2000Component *comp,
Jpeg2000CodingStyle *codsty)
for (precno = 0; precno < reslevel->num_precincts_x *
reslevel->num_precincts_y; precno++) {
if (band->prec) {
Jpeg2000Prec *prec = band->prec + precno;
+int nb_code_blocks = prec->nb_codeblocks_height *
prec->nb_codeblocks_width;
+
av_freep(&prec->zerobits);
av_freep(&prec->cblkincl);
-av_freep(&prec->cblk);
+if (prec->cblk) {
+int cblkno;
+for (cblkno = 0; cblkno < nb_code_blocks; cblkno ++) {
+Jpeg2000Cblk *cblk = &prec->cblk[cblkno];
+av_freep(&cblk->data);
+}
+av_freep(&prec->cblk);
+}
}
}
diff --git a/libavcodec/jpeg2000.h b/libavcodec/jpeg2000.h
index 8a022ad918..eaf7faf342 100644
--- a/libavcodec/jpeg2000.h
+++ b/libavcodec/jpeg2000.h
@@ -168,8 +168,8 @@ typedef struct Jpeg2000Cblk {
uint16_t lengthinc[JPEG2000_MAX_PASSES];
uint8_t nb_lengthinc;
uint8_t lblock;
-uint8_t zero;
-uint8_t data[8192];
+uint8_t *data;
+size_t data_allocated;
int nb_terminations;
int nb_terminationsinc;
int data_start[JPEG2000_MAX_PASSES];
diff --git a/libavcodec/jpeg2000dec.c b/libavcodec/jpeg2000dec.c
index 62b9009a82..0309b1f6fb 100644
--- a/libavcodec/jpeg2000dec.c
+++ b/libavcodec/jpeg2000dec.c
@@ -1001,10 +1001,18 @@ static int
jpeg2000_decode_packet(Jpeg2000DecoderContext *s, Jpeg2000Tile *tile,
if ((ret = get_bits(s, av_log2(newpasses1) + cblk->lblock)) <
0)
return ret;
-if (ret > sizeof(cblk->data)) {
+if (ret > cblk->data_allocated) {
+size_t new_size = FFMAX(2*cblk->data_allocated, ret);
+void *new = av_realloc(cblk->data, new_size);
+if (new) {
+cblk->data = new;
+