From: Mashiat Sarker Shakkhar <[email protected]>
Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind
CC: [email protected]
Signed-off-by: Ronald S. Bultje <[email protected]>
---
libavcodec/vc1.c | 2 +-
libavcodec/vc1dec.c | 36 +++++++++++++++++++++++++++++-------
2 files changed, 30 insertions(+), 8 deletions(-)
diff --git a/libavcodec/vc1.c b/libavcodec/vc1.c
index 433baa9..bb0b775 100644
--- a/libavcodec/vc1.c
+++ b/libavcodec/vc1.c
@@ -492,7 +492,7 @@ static int decode_sequence_header_adv(VC1Context *v,
GetBitContext *gb)
int nr, dr;
nr = get_bits(gb, 8);
dr = get_bits(gb, 4);
- if (nr && nr < 8 && dr && dr < 3) {
+ if (nr > 0 && nr < 6 && dr > 0 && dr < 3) {
v->s.avctx->time_base.num = ff_vc1_fps_dr[dr - 1];
v->s.avctx->time_base.den = ff_vc1_fps_nr[nr - 1] * 1000;
}
diff --git a/libavcodec/vc1dec.c b/libavcodec/vc1dec.c
index a08ad11..6960aff 100644
--- a/libavcodec/vc1dec.c
+++ b/libavcodec/vc1dec.c
@@ -2370,6 +2370,7 @@ static inline int vc1_pred_dc(MpegEncContext *s, int
overlap, int pq, int n,
int16_t *dc_val;
int mb_pos = s->mb_x + s->mb_y * s->mb_stride;
int q1, q2 = 0;
+ int dqscale_index;
wrap = s->block_wrap[n];
dc_val = s->dc_val[0] + s->block_index[n];
@@ -2382,15 +2383,18 @@ static inline int vc1_pred_dc(MpegEncContext *s, int
overlap, int pq, int n,
a = dc_val[ - wrap];
/* scale predictors if needed */
q1 = s->current_picture.f.qscale_table[mb_pos];
+ dqscale_index = s->y_dc_scale_table[q1] - 1;
+ if (dqscale_index < 0)
+ return AVERROR_INVALIDDATA;
if (c_avail && (n != 1 && n != 3)) {
q2 = s->current_picture.f.qscale_table[mb_pos - 1];
if (q2 && q2 != q1)
- c = (c * s->y_dc_scale_table[q2] *
ff_vc1_dqscale[s->y_dc_scale_table[q1] - 1] + 0x20000) >> 18;
+ c = (c * s->y_dc_scale_table[q2] * ff_vc1_dqscale[dqscale_index] +
0x20000) >> 18;
}
if (a_avail && (n != 2 && n != 3)) {
q2 = s->current_picture.f.qscale_table[mb_pos - s->mb_stride];
if (q2 && q2 != q1)
- a = (a * s->y_dc_scale_table[q2] *
ff_vc1_dqscale[s->y_dc_scale_table[q1] - 1] + 0x20000) >> 18;
+ a = (a * s->y_dc_scale_table[q2] * ff_vc1_dqscale[dqscale_index] +
0x20000) >> 18;
}
if (a_avail && c_avail && (n != 3)) {
int off = mb_pos;
@@ -2400,7 +2404,7 @@ static inline int vc1_pred_dc(MpegEncContext *s, int
overlap, int pq, int n,
off -= s->mb_stride;
q2 = s->current_picture.f.qscale_table[off];
if (q2 && q2 != q1)
- b = (b * s->y_dc_scale_table[q2] *
ff_vc1_dqscale[s->y_dc_scale_table[q1] - 1] + 0x20000) >> 18;
+ b = (b * s->y_dc_scale_table[q2] * ff_vc1_dqscale[dqscale_index] +
0x20000) >> 18;
}
if (a_avail && c_avail) {
@@ -2709,7 +2713,7 @@ static int vc1_decode_i_block_adv(VC1Context *v, DCTELEM
block[64], int n,
int i;
int16_t *dc_val;
int16_t *ac_val, *ac_val2;
- int dcdiff;
+ int dcdiff, pred;
int a_avail = v->a_avail, c_avail = v->c_avail;
int use_pred = s->ac_pred;
int scale;
@@ -2743,7 +2747,10 @@ static int vc1_decode_i_block_adv(VC1Context *v, DCTELEM
block[64], int n,
}
/* Prediction */
- dcdiff += vc1_pred_dc(&v->s, v->overlap, mquant, n, v->a_avail,
v->c_avail, &dc_val, &dc_pred_dir);
+ pred = vc1_pred_dc(&v->s, v->overlap, mquant, n, v->a_avail, v->c_avail,
&dc_val, &dc_pred_dir);
+ if (pred < 0)
+ return pred;
+ dcdiff += pred;
*dc_val = dcdiff;
/* Store the quantized DC coeff, used for prediction */
@@ -2817,6 +2824,8 @@ static int vc1_decode_i_block_adv(VC1Context *v, DCTELEM
block[64], int n,
q1 = q1 * 2 + ((q1 == v->pq) ? v->halfpq : 0) - 1;
q2 = q2 * 2 + ((q2 == v->pq) ? v->halfpq : 0) - 1;
+ if (q1 < 1)
+ return AVERROR_INVALIDDATA;
if (dc_pred_dir) { // left
for (k = 1; k < 8; k++)
block[k << v->left_blk_sh] += (ac_val[k] * q2 *
ff_vc1_dqscale[q1 - 1] + 0x20000) >> 18;
@@ -2859,6 +2868,8 @@ static int vc1_decode_i_block_adv(VC1Context *v, DCTELEM
block[64], int n,
if (q2 && q1 != q2) {
q1 = q1 * 2 + ((q1 == v->pq) ? v->halfpq : 0) - 1;
q2 = q2 * 2 + ((q2 == v->pq) ? v->halfpq : 0) - 1;
+ if (q1 < 1)
+ return AVERROR_INVALIDDATA;
for (k = 1; k < 8; k++)
ac_val2[k] = (ac_val2[k] * q2 * ff_vc1_dqscale[q1 - 1]
+ 0x20000) >> 18;
}
@@ -2869,6 +2880,8 @@ static int vc1_decode_i_block_adv(VC1Context *v, DCTELEM
block[64], int n,
if (q2 && q1 != q2) {
q1 = q1 * 2 + ((q1 == v->pq) ? v->halfpq : 0) - 1;
q2 = q2 * 2 + ((q2 == v->pq) ? v->halfpq : 0) - 1;
+ if (q1 < 1)
+ return AVERROR_INVALIDDATA;
for (k = 1; k < 8; k++)
ac_val2[k + 8] = (ac_val2[k + 8] * q2 *
ff_vc1_dqscale[q1 - 1] + 0x20000) >> 18;
}
@@ -2915,7 +2928,7 @@ static int vc1_decode_intra_block(VC1Context *v, DCTELEM
block[64], int n,
int i;
int16_t *dc_val;
int16_t *ac_val, *ac_val2;
- int dcdiff;
+ int dcdiff, pred;
int mb_pos = s->mb_x + s->mb_y * s->mb_stride;
int a_avail = v->a_avail, c_avail = v->c_avail;
int use_pred = s->ac_pred;
@@ -2958,7 +2971,10 @@ static int vc1_decode_intra_block(VC1Context *v, DCTELEM
block[64], int n,
}
/* Prediction */
- dcdiff += vc1_pred_dc(&v->s, v->overlap, mquant, n, a_avail, c_avail,
&dc_val, &dc_pred_dir);
+ pred = vc1_pred_dc(&v->s, v->overlap, mquant, n, a_avail, c_avail,
&dc_val, &dc_pred_dir);
+ if (pred < 0)
+ return pred;
+ dcdiff += pred;
*dc_val = dcdiff;
/* Store the quantized DC coeff, used for prediction */
@@ -3027,6 +3043,8 @@ static int vc1_decode_intra_block(VC1Context *v, DCTELEM
block[64], int n,
q1 = q1 * 2 + ((q1 == v->pq) ? v->halfpq : 0) - 1;
q2 = q2 * 2 + ((q2 == v->pq) ? v->halfpq : 0) - 1;
+ if (q1 < 1)
+ return AVERROR_INVALIDDATA;
if (dc_pred_dir) { // left
for (k = 1; k < 8; k++)
block[k << v->left_blk_sh] += (ac_val[k] * q2 *
ff_vc1_dqscale[q1 - 1] + 0x20000) >> 18;
@@ -3069,6 +3087,8 @@ static int vc1_decode_intra_block(VC1Context *v, DCTELEM
block[64], int n,
if (q2 && q1 != q2) {
q1 = q1 * 2 + ((q1 == v->pq) ? v->halfpq : 0) - 1;
q2 = q2 * 2 + ((q2 == v->pq) ? v->halfpq : 0) - 1;
+ if (q1 < 1)
+ return AVERROR_INVALIDDATA;
for (k = 1; k < 8; k++)
ac_val2[k] = (ac_val2[k] * q2 * ff_vc1_dqscale[q1 - 1]
+ 0x20000) >> 18;
}
@@ -3079,6 +3099,8 @@ static int vc1_decode_intra_block(VC1Context *v, DCTELEM
block[64], int n,
if (q2 && q1 != q2) {
q1 = q1 * 2 + ((q1 == v->pq) ? v->halfpq : 0) - 1;
q2 = q2 * 2 + ((q2 == v->pq) ? v->halfpq : 0) - 1;
+ if (q1 < 1)
+ return AVERROR_INVALIDDATA;
for (k = 1; k < 8; k++)
ac_val2[k + 8] = (ac_val2[k + 8] * q2 *
ff_vc1_dqscale[q1 - 1] + 0x20000) >> 18;
}
--
1.7.9.2
_______________________________________________
libav-devel mailing list
[email protected]
https://lists.libav.org/mailman/listinfo/libav-devel
