Module: libav Branch: master Commit: aacf6b3a2fd8bc8603e3deaa6e612ea03cf08707
Author: John Brooks <[email protected]> Committer: Kostya Shishkov <[email protected]> Date: Tue Jan 3 23:27:35 2012 -0700 rv34: fix invalid memory access for small video dimensions For small video dimensions calculations of the upper bound for pixel access may result in negative value. Using an unsigned comparison works only if the greater operand is non-negative. This is fixed by doing edge emulation explicitly for such conditions. Signed-off-by: Kostya Shishkov <[email protected]> --- libavcodec/rv34.c | 5 +++-- 1 files changed, 3 insertions(+), 2 deletions(-) diff --git a/libavcodec/rv34.c b/libavcodec/rv34.c index db2e53a..91d6788 100644 --- a/libavcodec/rv34.c +++ b/libavcodec/rv34.c @@ -680,8 +680,9 @@ static inline void rv34_mc(RV34DecContext *r, const int block_type, srcY += src_y * s->linesize + src_x; srcU += uvsrc_y * s->uvlinesize + uvsrc_x; srcV += uvsrc_y * s->uvlinesize + uvsrc_x; - if( (unsigned)(src_x - !!lx*2) > s->h_edge_pos - !!lx*2 - (width <<3) - 4 - || (unsigned)(src_y - !!ly*2) > s->v_edge_pos - !!ly*2 - (height<<3) - 4){ + if(s->h_edge_pos - (width << 3) < 6 || s->v_edge_pos - (height << 3) < 6 || + (unsigned)(src_x - !!lx*2) > s->h_edge_pos - !!lx*2 - (width <<3) - 4 || + (unsigned)(src_y - !!ly*2) > s->v_edge_pos - !!ly*2 - (height<<3) - 4) { uint8_t *uvbuf = s->edge_emu_buffer + 22 * s->linesize; srcY -= 2 + 2*s->linesize; _______________________________________________ libav-commits mailing list [email protected] https://lists.libav.org/mailman/listinfo/libav-commits
