vlc | branch: master | Rémi Denis-Courmont <[email protected]> | Thu Sep 7 21:54:37 2017 +0300| [0a0690cc4a4b009de856143378cfebfe2b7a680f] | committer: Rémi Denis-Courmont
gradient: fix integer overflow If a and/or b is unsigned, a - b is unsigned. Then if a < b, a - b > INT_MAX. And then abs(a - b) performs an undefined implicit conversion to int. This converts to int before computing the difference. Since all samples are 8-bits unsigned, this cannot overflow. > http://git.videolan.org/gitweb.cgi/vlc.git/?a=commit;h=0a0690cc4a4b009de856143378cfebfe2b7a680f --- modules/video_filter/gradient.c | 65 +++++++++++++++++------------------------ 1 file changed, 26 insertions(+), 39 deletions(-) diff --git a/modules/video_filter/gradient.c b/modules/video_filter/gradient.c index 55a84e9990..7479377d8d 100644 --- a/modules/video_filter/gradient.c +++ b/modules/video_filter/gradient.c @@ -382,26 +382,20 @@ static void FilterGradient( filter_t *p_filter, picture_t *p_inpic, { \ for( int x = 1; x < i_src_visible - 1; x++ ) \ { \ - const uint32_t a = \ - ( \ - abs( \ - ( p_smooth[(y-1)*i_src_visible+x-1] \ - - p_smooth[(y+1)*i_src_visible+x-1] ) \ - + ( ( p_smooth[(y-1)*i_src_visible+x] \ - - p_smooth[(y+1)*i_src_visible+x] ) <<1 ) \ - + ( p_smooth[(y-1)*i_src_visible+x+1] \ - - p_smooth[(y+1)*i_src_visible+x+1] ) \ - ) \ - + \ - abs( \ - ( p_smooth[(y-1)*i_src_visible+x-1] \ - - p_smooth[(y-1)*i_src_visible+x+1] ) \ - + ( ( p_smooth[y*i_src_visible+x-1] \ - - p_smooth[y*i_src_visible+x+1] ) <<1 ) \ - + ( p_smooth[(y+1)*i_src_visible+x-1] \ - - p_smooth[(y+1)*i_src_visible+x+1] ) \ - ) \ - ); + const uint32_t a = \ + abs(((int)p_smooth[(y - 1) * i_src_visible + x - 1] \ + - (int)p_smooth[(y + 1) * i_src_visible + x - 1]) \ + + (((int)p_smooth[(y - 1) * i_src_visible + x] \ + - (int)p_smooth[(y + 1) * i_src_visible + x]) * 2) \ + + ((int)p_smooth[(y - 1) * i_src_visible + x + 1] \ + - (int)p_smooth[(y + 1) * i_src_visible + x + 1])) \ + + abs(((int)p_smooth[(y - 1) * i_src_visible + x - 1] \ + - (int)p_smooth[(y - 1) * i_src_visible + x + 1]) \ + + (((int)p_smooth[y * i_src_visible + x - 1] \ + - (int)p_smooth[y * i_src_visible + x + 1]) * 2) \ + + ((int)p_smooth[(y + 1) * i_src_visible + x - 1] \ + - (int)p_smooth[(y + 1) * i_src_visible + x + 1])); + if( p_filter->p_sys->i_gradient_type ) { if( p_filter->p_sys->b_cartoon ) @@ -694,25 +688,18 @@ static void FilterHough( filter_t *p_filter, picture_t *p_inpic, for( int x = 4; x < i_src_visible - 4; x++ ) { uint32_t a = - ( - abs( - ( ( p_smooth[(y-1)*i_src_visible+x] - - p_smooth[(y+1)*i_src_visible+x] ) <<1 ) - + ( p_smooth[(y-1)*i_src_visible+x-1] - - p_smooth[(y+1)*i_src_visible+x-1] ) - + ( p_smooth[(y-1)*i_src_visible+x+1] - - p_smooth[(y+1)*i_src_visible+x+1] ) - ) - + - abs( - ( ( p_smooth[y*i_src_visible+x-1] - - p_smooth[y*i_src_visible+x+1] ) <<1 ) - + ( p_smooth[(y-1)*i_src_visible+x-1] - - p_smooth[(y-1)*i_src_visible+x+1] ) - + ( p_smooth[(y+1)*i_src_visible+x-1] - - p_smooth[(y+1)*i_src_visible+x+1] ) - ) - ); + abs((((int)p_smooth[(y - 1) * i_src_visible + x] + - (int)p_smooth[(y + 1) * i_src_visible + x]) * 2) + + ((int)p_smooth[(y - 1) * i_src_visible + x - 1] + - (int)p_smooth[(y + 1) * i_src_visible + x - 1]) + + ((int)p_smooth[(y - 1) * i_src_visible + x + 1] + - (int)p_smooth[(y + 1) * i_src_visible + x + 1])) + + abs((((int)p_smooth[y * i_src_visible + x - 1] + - (int)p_smooth[y * i_src_visible + x + 1]) * 2) + + ((int)p_smooth[(y - 1) * i_src_visible + x - 1] + - (int)p_smooth[(y - 1) * i_src_visible + x + 1]) + + ((int)p_smooth[(y + 1) * i_src_visible + x - 1] + - (int)p_smooth[(y + 1) * i_src_visible + x + 1])); if( a>>8 ) { for( int i = 0; i < i_nb_steps; i++ ) _______________________________________________ vlc-commits mailing list [email protected] https://mailman.videolan.org/listinfo/vlc-commits
