From: Michael Niedermayer <[email protected]> If the src_y variable is very large, the multiplication src_y * linesize might overflow. The same multiplication is done in the caller (e.g. mpeg_motion_internal) where both src_y and linesize are ints, where part of this offsetting is undone within emulated_edge_mc. If this multplication overflowed in the caller, make sure it overflows in the same way here.
This is mostly relevant for e.g. h263+ where the motion vectors can be unrestricted. Reported-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind CC: [email protected] --- libavcodec/videodsp_template.c | 9 ++++++--- libavcodec/x86/videodsp_init.c | 9 ++++++--- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/libavcodec/videodsp_template.c b/libavcodec/videodsp_template.c index 5d5a32f..334c27b 100644 --- a/libavcodec/videodsp_template.c +++ b/libavcodec/videodsp_template.c @@ -22,18 +22,21 @@ #include "bit_depth_template.c" static void FUNC(ff_emulated_edge_mc)(uint8_t *buf, const uint8_t *src, - ptrdiff_t linesize, + ptrdiff_t linesize_arg, int block_w, int block_h, int src_x, int src_y, int w, int h) { int x, y; int start_y, start_x, end_y, end_x; + int linesize = linesize_arg; if (src_y >= h) { - src += (h - 1 - src_y) * linesize; + src -= src_y * linesize; + src += (h - 1) * linesize; src_y = h - 1; } else if (src_y <= -block_h) { - src += (1 - block_h - src_y) * linesize; + src -= src_y * linesize; + src += (1 - block_h) * linesize; src_y = 1 - block_h; } if (src_x >= w) { diff --git a/libavcodec/x86/videodsp_init.c b/libavcodec/x86/videodsp_init.c index f7fd267..688a894 100644 --- a/libavcodec/x86/videodsp_init.c +++ b/libavcodec/x86/videodsp_init.c @@ -37,19 +37,22 @@ extern emu_edge_core_func ff_emu_edge_core_mmx; extern emu_edge_core_func ff_emu_edge_core_sse; static av_always_inline void emulated_edge_mc(uint8_t *buf, const uint8_t *src, - ptrdiff_t linesize, + ptrdiff_t linesize_arg, int block_w, int block_h, int src_x, int src_y, int w, int h, emu_edge_core_func *core_fn) { int start_y, start_x, end_y, end_x, src_y_add = 0; + int linesize = linesize_arg; if (src_y >= h) { - src_y_add = h - 1 - src_y; + src -= src_y * linesize; + src_y_add = h - 1; src_y = h - 1; } else if (src_y <= -block_h) { - src_y_add = 1 - block_h - src_y; + src -= src_y * linesize; + src_y_add = 1 - block_h; src_y = 1 - block_h; } if (src_x >= w) { -- 1.7.9.4 _______________________________________________ libav-devel mailing list [email protected] https://lists.libav.org/mailman/listinfo/libav-devel
