For small video dimensions, these calculations of the upper bound
for pixel access may have a negative result. Using an unsigned
comparison to bound a potentially negative value only works if
the greater operand is non-negative. Fixed by doing edge emulation
when the upper bound is probably negative, everywhere that this
pattern appears.
---
libavcodec/mpegvideo.c | 9 ++++++---
libavcodec/mpegvideo_common.h | 35 +++++++++++++++++++++--------------
2 files changed, 27 insertions(+), 17 deletions(-)
diff --git a/libavcodec/mpegvideo.c b/libavcodec/mpegvideo.c
index a2aa257..9601bb2 100644
--- a/libavcodec/mpegvideo.c
+++ b/libavcodec/mpegvideo.c
@@ -1834,7 +1834,8 @@ static inline int hpel_motion_lowres(MpegEncContext *s,
src += src_y * stride + src_x;
- if ((unsigned)src_x > h_edge_pos - (!!sx) - w ||
+ if (h_edge_pos <= w || (v_edge_pos >> field_based) <= h ||
+ (unsigned)src_x > h_edge_pos - (!!sx) - w ||
(unsigned)src_y > (v_edge_pos >> field_based) - (!!sy) - h) {
s->dsp.emulated_edge_mc(s->edge_emu_buffer, src, s->linesize, w + 1,
(h + 1) << field_based, src_x,
@@ -1919,7 +1920,8 @@ static av_always_inline void
mpeg_motion_lowres(MpegEncContext *s,
ptr_cb = ref_picture[1] + uvsrc_y * uvlinesize + uvsrc_x;
ptr_cr = ref_picture[2] + uvsrc_y * uvlinesize + uvsrc_x;
- if ((unsigned) src_x > h_edge_pos - (!!sx) - 2 * block_s ||
+ if (h_edge_pos <= 2 * block_s || (v_edge_pos >> field_based) <= h ||
+ (unsigned) src_x > h_edge_pos - (!!sx) - 2 * block_s ||
(unsigned) src_y > (v_edge_pos >> field_based) - (!!sy) - h) {
s->dsp.emulated_edge_mc(s->edge_emu_buffer, ptr_y,
s->linesize, 17, 17 + field_based,
@@ -2002,7 +2004,8 @@ static inline void
chroma_4mv_motion_lowres(MpegEncContext *s,
offset = src_y * s->uvlinesize + src_x;
ptr = ref_picture[1] + offset;
if (s->flags & CODEC_FLAG_EMU_EDGE) {
- if ((unsigned) src_x > h_edge_pos - (!!sx) - block_s ||
+ if (h_edge_pos <= block_s || v_edge_pos <= block_s ||
+ (unsigned) src_x > h_edge_pos - (!!sx) - block_s ||
(unsigned) src_y > v_edge_pos - (!!sy) - block_s) {
s->dsp.emulated_edge_mc(s->edge_emu_buffer, ptr, s->uvlinesize,
9, 9, src_x, src_y, h_edge_pos,
v_edge_pos);
diff --git a/libavcodec/mpegvideo_common.h b/libavcodec/mpegvideo_common.h
index d64404d..a731921 100644
--- a/libavcodec/mpegvideo_common.h
+++ b/libavcodec/mpegvideo_common.h
@@ -81,8 +81,9 @@ static inline void gmc1_motion(MpegEncContext *s,
ptr = ref_picture[0] + (src_y * linesize) + src_x;
if(s->flags&CODEC_FLAG_EMU_EDGE){
- if( (unsigned)src_x >= s->h_edge_pos - 17
- || (unsigned)src_y >= s->v_edge_pos - 17){
+ if(s->h_edge_pos < 17 || s->v_edge_pos < 17 ||
+ (unsigned)src_x >= s->h_edge_pos - 17 ||
+ (unsigned)src_y >= s->v_edge_pos - 17) {
s->dsp.emulated_edge_mc(s->edge_emu_buffer, ptr, linesize, 17, 17,
src_x, src_y, s->h_edge_pos, s->v_edge_pos);
ptr= s->edge_emu_buffer;
}
@@ -120,8 +121,9 @@ static inline void gmc1_motion(MpegEncContext *s,
offset = (src_y * uvlinesize) + src_x;
ptr = ref_picture[1] + offset;
if(s->flags&CODEC_FLAG_EMU_EDGE){
- if( (unsigned)src_x >= (s->h_edge_pos>>1) - 9
- || (unsigned)src_y >= (s->v_edge_pos>>1) - 9){
+ if(s->h_edge_pos < 18 || s->v_edge_pos < 18 ||
+ (unsigned)src_x >= (s->h_edge_pos>>1) - 9 ||
+ (unsigned)src_y >= (s->v_edge_pos>>1) - 9) {
s->dsp.emulated_edge_mc(s->edge_emu_buffer, ptr, uvlinesize, 9, 9,
src_x, src_y, s->h_edge_pos>>1, s->v_edge_pos>>1);
ptr= s->edge_emu_buffer;
emu=1;
@@ -221,8 +223,9 @@ static inline int hpel_motion(MpegEncContext *s,
src += src_y * stride + src_x;
if(s->unrestricted_mv && (s->flags&CODEC_FLAG_EMU_EDGE)){
- if( (unsigned)src_x > h_edge_pos - (motion_x&1) - w
- || (unsigned)src_y > v_edge_pos - (motion_y&1) - h){
+ if(h_edge_pos <= w || v_edge_pos <= h ||
+ (unsigned)src_x > h_edge_pos - (motion_x&1) - w ||
+ (unsigned)src_y > v_edge_pos - (motion_y&1) - h) {
s->dsp.emulated_edge_mc(s->edge_emu_buffer, src, s->linesize, w+1,
(h+1)<<field_based,
src_x, src_y<<field_based, h_edge_pos,
s->v_edge_pos);
src= s->edge_emu_buffer;
@@ -307,8 +310,9 @@ if(s->quarter_sample)
ptr_cb = ref_picture[1] + uvsrc_y * uvlinesize + uvsrc_x;
ptr_cr = ref_picture[2] + uvsrc_y * uvlinesize + uvsrc_x;
- if( (unsigned)src_x > s->h_edge_pos - (motion_x&1) - 16
- || (unsigned)src_y > v_edge_pos - (motion_y&1) - h){
+ if(s->h_edge_pos <= 16 || v_edge_pos <= h ||
+ (unsigned)src_x > s->h_edge_pos - (motion_x&1) - 16 ||
+ (unsigned)src_y > v_edge_pos - (motion_y&1) - h) {
if(is_mpeg12 || s->codec_id == CODEC_ID_MPEG2VIDEO ||
s->codec_id == CODEC_ID_MPEG1VIDEO){
av_log(s->avctx,AV_LOG_DEBUG,
@@ -510,8 +514,9 @@ static inline void qpel_motion(MpegEncContext *s,
ptr_cb = ref_picture[1] + uvsrc_y * uvlinesize + uvsrc_x;
ptr_cr = ref_picture[2] + uvsrc_y * uvlinesize + uvsrc_x;
- if( (unsigned)src_x > s->h_edge_pos - (motion_x&3) - 16
- || (unsigned)src_y > v_edge_pos - (motion_y&3) - h ){
+ if(s->h_edge_pos - 16 < 3 || v_edge_pos - h < 3 ||
+ (unsigned)src_x > s->h_edge_pos - (motion_x&3) - 16 ||
+ (unsigned)src_y > v_edge_pos - (motion_y&3) - h ) {
s->dsp.emulated_edge_mc(s->edge_emu_buffer, ptr_y, s->linesize,
17, 17+field_based, src_x, src_y<<field_based,
s->h_edge_pos, s->v_edge_pos);
@@ -588,8 +593,9 @@ static inline void chroma_4mv_motion(MpegEncContext *s,
offset = src_y * s->uvlinesize + src_x;
ptr = ref_picture[1] + offset;
if(s->flags&CODEC_FLAG_EMU_EDGE){
- if( (unsigned)src_x > (s->h_edge_pos>>1) - (dxy &1) - 8
- || (unsigned)src_y > (s->v_edge_pos>>1) - (dxy>>1) - 8){
+ if(s->h_edge_pos <= 16 || s->v_edge_pos - dxy <= 16 ||
+ (unsigned)src_x > (s->h_edge_pos>>1) - (dxy &1) - 8 ||
+ (unsigned)src_y > (s->v_edge_pos>>1) - (dxy>>1) - 8) {
s->dsp.emulated_edge_mc(s->edge_emu_buffer, ptr, s->uvlinesize,
9, 9, src_x, src_y,
s->h_edge_pos>>1, s->v_edge_pos>>1);
@@ -760,8 +766,9 @@ static av_always_inline void
MPV_motion_internal(MpegEncContext *s,
ptr = ref_picture[0] + (src_y * s->linesize) + (src_x);
if(s->flags&CODEC_FLAG_EMU_EDGE){
- if( (unsigned)src_x > s->h_edge_pos - (motion_x&3) - 8
- || (unsigned)src_y > s->v_edge_pos - (motion_y&3) - 8 ){
+ if(s->h_edge_pos < 11 || s->v_edge_pos < 11 ||
+ (unsigned)src_x > s->h_edge_pos - (motion_x&3) - 8 ||
+ (unsigned)src_y > s->v_edge_pos - (motion_y&3) - 8) {
s->dsp.emulated_edge_mc(s->edge_emu_buffer, ptr,
s->linesize, 9, 9,
src_x, src_y,
--
1.7.5.4
_______________________________________________
libav-devel mailing list
[email protected]
https://lists.libav.org/mailman/listinfo/libav-devel
