Module: libav Branch: release/0.7 Commit: 047c6ad752386e892afd45fd97214108e303776f
Author: Ronald S. Bultje <[email protected]> Committer: Reinhard Tartler <[email protected]> Date: Thu Feb 9 22:57:01 2012 -0800 h264: disallow constrained intra prediction modes for luma. Conversion of the luma intra prediction mode to one of the constrained ("alzheimer") ones can happen by crafting special bitstreams, causing a crash because we'll call a NULL function pointer for 16x16 block intra prediction, since constrained intra prediction functions are only implemented for chroma (8x8 blocks). Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind CC: [email protected] (cherry picked from commit 45b7bd7c53b41bc5ff6fc2158831f2b1b1256113) Signed-off-by: Reinhard Tartler <[email protected]> (cherry picked from commit 248d4e461578ff327a2fd75fd0db4f38c270918a) Signed-off-by: Reinhard Tartler <[email protected]> --- libavcodec/h264.c | 4 ++-- libavcodec/h264.h | 2 +- libavcodec/h264_cabac.c | 4 ++-- libavcodec/h264_cavlc.c | 4 ++-- libavcodec/svq3.c | 4 ++-- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/libavcodec/h264.c b/libavcodec/h264.c index c9187e6..dbf8e6a 100644 --- a/libavcodec/h264.c +++ b/libavcodec/h264.c @@ -111,7 +111,7 @@ int ff_h264_check_intra4x4_pred_mode(H264Context *h){ /** * checks if the top & left blocks are available if needed & changes the dc mode so it only uses the available blocks. */ -int ff_h264_check_intra_pred_mode(H264Context *h, int mode){ +int ff_h264_check_intra_pred_mode(H264Context *h, int mode, int is_chroma){ MpegEncContext * const s = &h->s; static const int8_t top [7]= {LEFT_DC_PRED8x8, 1,-1,-1}; static const int8_t left[7]= { TOP_DC_PRED8x8,-1, 2,-1,DC_128_PRED8x8}; @@ -131,7 +131,7 @@ int ff_h264_check_intra_pred_mode(H264Context *h, int mode){ if((h->left_samples_available&0x8080) != 0x8080){ mode= left[ mode ]; - if(h->left_samples_available&0x8080){ //mad cow disease mode, aka MBAFF + constrained_intra_pred + if(is_chroma && (h->left_samples_available&0x8080)){ //mad cow disease mode, aka MBAFF + constrained_intra_pred mode= ALZHEIMER_DC_L0T_PRED8x8 + (!(h->left_samples_available&0x8000)) + 2*(mode == DC_128_PRED8x8); } if(mode<0){ diff --git a/libavcodec/h264.h b/libavcodec/h264.h index e3cc815..507e78f 100644 --- a/libavcodec/h264.h +++ b/libavcodec/h264.h @@ -656,7 +656,7 @@ int ff_h264_check_intra4x4_pred_mode(H264Context *h); /** * Check if the top & left blocks are available if needed & change the dc mode so it only uses the available blocks. */ -int ff_h264_check_intra_pred_mode(H264Context *h, int mode); +int ff_h264_check_intra_pred_mode(H264Context *h, int mode, int is_chroma); void ff_h264_write_back_intra_pred_mode(H264Context *h); void ff_h264_hl_decode_mb(H264Context *h); diff --git a/libavcodec/h264_cabac.c b/libavcodec/h264_cabac.c index f30f4e1..e8c8503 100644 --- a/libavcodec/h264_cabac.c +++ b/libavcodec/h264_cabac.c @@ -2002,14 +2002,14 @@ decode_intra_mb: ff_h264_write_back_intra_pred_mode(h); if( ff_h264_check_intra4x4_pred_mode(h) < 0 ) return -1; } else { - h->intra16x16_pred_mode= ff_h264_check_intra_pred_mode( h, h->intra16x16_pred_mode ); + h->intra16x16_pred_mode= ff_h264_check_intra_pred_mode( h, h->intra16x16_pred_mode, 0 ); if( h->intra16x16_pred_mode < 0 ) return -1; } if(decode_chroma){ h->chroma_pred_mode_table[mb_xy] = pred_mode = decode_cabac_mb_chroma_pre_mode( h ); - pred_mode= ff_h264_check_intra_pred_mode( h, pred_mode ); + pred_mode= ff_h264_check_intra_pred_mode( h, pred_mode, 1 ); if( pred_mode < 0 ) return -1; h->chroma_pred_mode= pred_mode; } else { diff --git a/libavcodec/h264_cavlc.c b/libavcodec/h264_cavlc.c index 2661ab2..cd4f336 100644 --- a/libavcodec/h264_cavlc.c +++ b/libavcodec/h264_cavlc.c @@ -736,12 +736,12 @@ decode_intra_mb: if( ff_h264_check_intra4x4_pred_mode(h) < 0) return -1; }else{ - h->intra16x16_pred_mode= ff_h264_check_intra_pred_mode(h, h->intra16x16_pred_mode); + h->intra16x16_pred_mode= ff_h264_check_intra_pred_mode(h, h->intra16x16_pred_mode, 0); if(h->intra16x16_pred_mode < 0) return -1; } if(decode_chroma){ - pred_mode= ff_h264_check_intra_pred_mode(h, get_ue_golomb_31(&s->gb)); + pred_mode= ff_h264_check_intra_pred_mode(h, get_ue_golomb_31(&s->gb), 1); if(pred_mode < 0) return -1; h->chroma_pred_mode= pred_mode; diff --git a/libavcodec/svq3.c b/libavcodec/svq3.c index 23ab209..09d598c 100644 --- a/libavcodec/svq3.c +++ b/libavcodec/svq3.c @@ -610,7 +610,7 @@ static int svq3_decode_mb(SVQ3Context *svq3, unsigned int mb_type) dir = i_mb_type_info[mb_type - 8].pred_mode; dir = (dir >> 1) ^ 3*(dir & 1) ^ 1; - if ((h->intra16x16_pred_mode = ff_h264_check_intra_pred_mode(h, dir)) == -1){ + if ((h->intra16x16_pred_mode = ff_h264_check_intra_pred_mode(h, dir, 0)) == -1){ av_log(h->s.avctx, AV_LOG_ERROR, "check_intra_pred_mode = -1\n"); return -1; } @@ -709,7 +709,7 @@ static int svq3_decode_mb(SVQ3Context *svq3, unsigned int mb_type) s->current_picture.mb_type[mb_xy] = mb_type; if (IS_INTRA(mb_type)) { - h->chroma_pred_mode = ff_h264_check_intra_pred_mode(h, DC_PRED8x8); + h->chroma_pred_mode = ff_h264_check_intra_pred_mode(h, DC_PRED8x8, 1); } return 0; _______________________________________________ libav-commits mailing list [email protected] https://lists.libav.org/mailman/listinfo/libav-commits
