For small video dimensions, these calculations of the upper bound
for pixel access may have a negative result. Using an unsigned
comparison to bound a potentially negative value only works if
the greater operand is non-negative. Fixed by doing edge emulation
when the upper bound is probably negative, everywhere that this
pattern appears.
---
libavcodec/rv34.c | 5 +++--
1 files changed, 3 insertions(+), 2 deletions(-)
diff --git a/libavcodec/rv34.c b/libavcodec/rv34.c
index 1a126be..f47a350 100644
--- a/libavcodec/rv34.c
+++ b/libavcodec/rv34.c
@@ -676,8 +676,9 @@ static inline void rv34_mc(RV34DecContext *r, const int
block_type,
srcY += src_y * s->linesize + src_x;
srcU += uvsrc_y * s->uvlinesize + uvsrc_x;
srcV += uvsrc_y * s->uvlinesize + uvsrc_x;
- if( (unsigned)(src_x - !!lx*2) > s->h_edge_pos - !!lx*2 - (width <<3) - 4
- || (unsigned)(src_y - !!ly*2) > s->v_edge_pos - !!ly*2 - (height<<3) -
4){
+ if(s->h_edge_pos - (width << 3) < 6 || s->v_edge_pos - (height << 3) < 6 ||
+ (unsigned)(src_x - !!lx*2) > s->h_edge_pos - !!lx*2 - (width <<3) - 4 ||
+ (unsigned)(src_y - !!ly*2) > s->v_edge_pos - !!ly*2 - (height<<3) - 4) {
uint8_t *uvbuf = s->edge_emu_buffer + 22 * s->linesize;
srcY -= 2 + 2*s->linesize;
--
1.7.5.4
_______________________________________________
libav-devel mailing list
[email protected]
https://lists.libav.org/mailman/listinfo/libav-devel
