vlc | branch: master | Francois Cartegnie <[email protected]> | Mon Sep 24 14:28:19 2018 +0200| [b3c301b6155a85eda8eef9d8a8d596c0e1d54783] | committer: Francois Cartegnie
video_converter: i420_rgb: lazy alloc conversion buffer Quote from A heap overflow world "4K ought to be enough for anybody" > http://git.videolan.org/gitweb.cgi/vlc.git/?a=commit;h=b3c301b6155a85eda8eef9d8a8d596c0e1d54783 --- modules/video_chroma/i420_rgb.c | 20 +++++------- modules/video_chroma/i420_rgb.h | 22 +++++++++++++ modules/video_chroma/i420_rgb16.c | 20 ++++++++++-- modules/video_chroma/i420_rgb16_x86.c | 58 +++++++++++++++++++++++++++++++---- 4 files changed, 99 insertions(+), 21 deletions(-) diff --git a/modules/video_chroma/i420_rgb.c b/modules/video_chroma/i420_rgb.c index 1c66223749..9f70694fe5 100644 --- a/modules/video_chroma/i420_rgb.c +++ b/modules/video_chroma/i420_rgb.c @@ -211,30 +211,26 @@ static int Activate( vlc_object_t *p_this ) return VLC_EGENERIC; p_filter->p_sys = p_sys; + p_sys->i_buffer_size = 0; + p_sys->p_buffer = NULL; switch( p_filter->fmt_out.video.i_chroma ) { #ifdef PLAIN case VLC_CODEC_RGB8: - p_sys->p_buffer = malloc( VOUT_MAX_WIDTH ); + p_sys->i_bytespp = 1; break; #endif case VLC_CODEC_RGB15: case VLC_CODEC_RGB16: - p_sys->p_buffer = malloc( VOUT_MAX_WIDTH * 2 ); + p_sys->i_bytespp = 2; break; case VLC_CODEC_RGB24: case VLC_CODEC_RGB32: - p_sys->p_buffer = malloc( VOUT_MAX_WIDTH * 4 ); + p_sys->i_bytespp = 4; break; default: - p_sys->p_buffer = NULL; - break; - } - - if( p_sys->p_buffer == NULL ) - { - free( p_sys ); - return VLC_EGENERIC; + free( p_sys ); + return VLC_EGENERIC; } p_sys->p_offset = malloc( p_filter->fmt_out.video.i_width @@ -243,7 +239,6 @@ static int Activate( vlc_object_t *p_this ) * sizeof( int ) ); if( p_sys->p_offset == NULL ) { - free( p_sys->p_buffer ); free( p_sys ); return VLC_EGENERIC; } @@ -267,7 +262,6 @@ static int Activate( vlc_object_t *p_this ) if( p_sys->p_base == NULL ) { free( p_sys->p_offset ); - free( p_sys->p_buffer ); free( p_sys ); return -1; } diff --git a/modules/video_chroma/i420_rgb.h b/modules/video_chroma/i420_rgb.h index 9bd156149e..6d060397b3 100644 --- a/modules/video_chroma/i420_rgb.h +++ b/modules/video_chroma/i420_rgb.h @@ -20,6 +20,7 @@ * along with this program; if not, write to the Free Software Foundation, * Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA. *****************************************************************************/ +#include <limits.h> #if !defined (SSE2) && !defined (MMX) # define PLAIN @@ -37,6 +38,8 @@ typedef struct { uint8_t *p_buffer; + size_t i_buffer_size; + uint8_t i_bytespp; int *p_offset; #ifdef PLAIN @@ -57,6 +60,25 @@ typedef struct } filter_sys_t; /***************************************************************************** + * Conversion buffer helper + *****************************************************************************/ +static inline int AllocateOrGrow( uint8_t **pp_buffer, size_t *pi_buffer, + unsigned i_width, uint8_t bytespp ) +{ + if(UINT_MAX / bytespp < i_width) + return -1; + const size_t i_realloc = i_width * bytespp; + if(*pi_buffer >= i_realloc) + return 0; + uint8_t *p_realloc = realloc(*pp_buffer, i_realloc); + if(!p_realloc) + return -1; + *pp_buffer = p_realloc; + *pi_buffer = i_realloc; + return 0; +} + +/***************************************************************************** * Prototypes *****************************************************************************/ #ifdef PLAIN diff --git a/modules/video_chroma/i420_rgb16.c b/modules/video_chroma/i420_rgb16.c index be37d8d355..85c7048ccd 100644 --- a/modules/video_chroma/i420_rgb16.c +++ b/modules/video_chroma/i420_rgb16.c @@ -130,7 +130,7 @@ void I420_RGB16( filter_t *p_filter, picture_t *p_src, picture_t *p_dest ) uint16_t * p_ybase; /* Y dependant conversion table */ /* Conversion buffer pointer */ - uint16_t * p_buffer_start = (uint16_t*)p_sys->p_buffer; + uint16_t * p_buffer_start; uint16_t * p_buffer; /* Offset array pointer */ @@ -156,6 +156,14 @@ void I420_RGB16( filter_t *p_filter, picture_t *p_src, picture_t *p_dest ) (p_filter->fmt_out.video.i_y_offset + p_filter->fmt_out.video.i_visible_height), &b_hscale, &i_vscale, p_offset_start ); + if(b_hscale && + AllocateOrGrow(&p_sys->p_buffer, &p_sys->i_buffer_size, + p_filter->fmt_in.video.i_x_offset + + p_filter->fmt_in.video.i_visible_width, + p_sys->i_bytespp)) + return; + else p_buffer_start = (uint16_t*)p_sys->p_buffer; + /* * Perform conversion */ @@ -237,7 +245,7 @@ void I420_RGB32( filter_t *p_filter, picture_t *p_src, picture_t *p_dest ) uint32_t * p_ybase; /* Y dependant conversion table */ /* Conversion buffer pointer */ - uint32_t * p_buffer_start = (uint32_t*)p_sys->p_buffer; + uint32_t * p_buffer_start; uint32_t * p_buffer; /* Offset array pointer */ @@ -263,6 +271,14 @@ void I420_RGB32( filter_t *p_filter, picture_t *p_src, picture_t *p_dest ) (p_filter->fmt_out.video.i_y_offset + p_filter->fmt_out.video.i_visible_height), &b_hscale, &i_vscale, p_offset_start ); + if(b_hscale && + AllocateOrGrow(&p_sys->p_buffer, &p_sys->i_buffer_size, + p_filter->fmt_in.video.i_x_offset + + p_filter->fmt_in.video.i_visible_width, + p_sys->i_bytespp)) + return; + else p_buffer_start = (uint32_t*)p_sys->p_buffer; + /* * Perform conversion */ diff --git a/modules/video_chroma/i420_rgb16_x86.c b/modules/video_chroma/i420_rgb16_x86.c index 37c425cf3d..ce39051443 100644 --- a/modules/video_chroma/i420_rgb16_x86.c +++ b/modules/video_chroma/i420_rgb16_x86.c @@ -122,7 +122,7 @@ void I420_R5G5B5( filter_t *p_filter, picture_t *p_src, picture_t *p_dest ) uint16_t * p_pic_start; /* beginning of the current line for copy */ /* Conversion buffer pointer */ - uint16_t * p_buffer_start = (uint16_t*)p_sys->p_buffer; + uint16_t * p_buffer_start; uint16_t * p_buffer; /* Offset array pointer */ @@ -147,6 +147,13 @@ void I420_R5G5B5( filter_t *p_filter, picture_t *p_src, picture_t *p_dest ) (p_filter->fmt_out.video.i_y_offset + p_filter->fmt_out.video.i_visible_height), &b_hscale, &i_vscale, p_offset_start ); + if(b_hscale && + AllocateOrGrow(&p_sys->p_buffer, &p_sys->i_buffer_size, + p_filter->fmt_in.video.i_x_offset + + p_filter->fmt_in.video.i_visible_width, + p_sys->i_bytespp)) + return; + else p_buffer_start = (uint16_t*)p_sys->p_buffer; /* * Perform conversion @@ -356,7 +363,7 @@ void I420_R5G6B5( filter_t *p_filter, picture_t *p_src, picture_t *p_dest ) uint16_t * p_pic_start; /* beginning of the current line for copy */ /* Conversion buffer pointer */ - uint16_t * p_buffer_start = (uint16_t*)p_sys->p_buffer; + uint16_t * p_buffer_start; uint16_t * p_buffer; /* Offset array pointer */ @@ -381,6 +388,13 @@ void I420_R5G6B5( filter_t *p_filter, picture_t *p_src, picture_t *p_dest ) (p_filter->fmt_out.video.i_y_offset + p_filter->fmt_out.video.i_visible_height), &b_hscale, &i_vscale, p_offset_start ); + if(b_hscale && + AllocateOrGrow(&p_sys->p_buffer, &p_sys->i_buffer_size, + p_filter->fmt_in.video.i_x_offset + + p_filter->fmt_in.video.i_visible_width, + p_sys->i_bytespp)) + return; + else p_buffer_start = (uint16_t*)p_sys->p_buffer; /* * Perform conversion @@ -590,7 +604,7 @@ void I420_A8R8G8B8( filter_t *p_filter, picture_t *p_src, int i_chroma_width = (p_filter->fmt_in.video.i_x_offset + p_filter->fmt_in.video.i_visible_width) / 2; /* chroma width */ uint32_t * p_pic_start; /* beginning of the current line for copy */ /* Conversion buffer pointer */ - uint32_t * p_buffer_start = (uint32_t*)p_sys->p_buffer; + uint32_t * p_buffer_start; uint32_t * p_buffer; /* Offset array pointer */ @@ -615,6 +629,14 @@ void I420_A8R8G8B8( filter_t *p_filter, picture_t *p_src, (p_filter->fmt_out.video.i_y_offset + p_filter->fmt_out.video.i_visible_height), &b_hscale, &i_vscale, p_offset_start ); + if(b_hscale && + AllocateOrGrow(&p_sys->p_buffer, &p_sys->i_buffer_size, + p_filter->fmt_in.video.i_x_offset + + p_filter->fmt_in.video.i_visible_width, + p_sys->i_bytespp)) + return; + else p_buffer_start = (uint32_t*)p_sys->p_buffer; + /* * Perform conversion */ @@ -822,7 +844,7 @@ void I420_R8G8B8A8( filter_t *p_filter, picture_t *p_src, picture_t *p_dest ) int i_chroma_width = (p_filter->fmt_in.video.i_x_offset + p_filter->fmt_in.video.i_visible_width) / 2; /* chroma width */ uint32_t * p_pic_start; /* beginning of the current line for copy */ /* Conversion buffer pointer */ - uint32_t * p_buffer_start = (uint32_t*)p_sys->p_buffer; + uint32_t * p_buffer_start; uint32_t * p_buffer; /* Offset array pointer */ @@ -847,6 +869,14 @@ void I420_R8G8B8A8( filter_t *p_filter, picture_t *p_src, picture_t *p_dest ) (p_filter->fmt_out.video.i_y_offset + p_filter->fmt_out.video.i_visible_height), &b_hscale, &i_vscale, p_offset_start ); + if(b_hscale && + AllocateOrGrow(&p_sys->p_buffer, &p_sys->i_buffer_size, + p_filter->fmt_in.video.i_x_offset + + p_filter->fmt_in.video.i_visible_width, + p_sys->i_bytespp)) + return; + else p_buffer_start = (uint32_t*)p_sys->p_buffer; + /* * Perform conversion */ @@ -1054,7 +1084,7 @@ void I420_B8G8R8A8( filter_t *p_filter, picture_t *p_src, picture_t *p_dest ) int i_chroma_width = (p_filter->fmt_in.video.i_x_offset + p_filter->fmt_in.video.i_visible_width) / 2; /* chroma width */ uint32_t * p_pic_start; /* beginning of the current line for copy */ /* Conversion buffer pointer */ - uint32_t * p_buffer_start = (uint32_t*)p_sys->p_buffer; + uint32_t * p_buffer_start; uint32_t * p_buffer; /* Offset array pointer */ @@ -1079,6 +1109,14 @@ void I420_B8G8R8A8( filter_t *p_filter, picture_t *p_src, picture_t *p_dest ) (p_filter->fmt_out.video.i_y_offset + p_filter->fmt_out.video.i_visible_height), &b_hscale, &i_vscale, p_offset_start ); + if(b_hscale && + AllocateOrGrow(&p_sys->p_buffer, &p_sys->i_buffer_size, + p_filter->fmt_in.video.i_x_offset + + p_filter->fmt_in.video.i_visible_width, + p_sys->i_bytespp)) + return; + else p_buffer_start = (uint32_t*)p_sys->p_buffer; + /* * Perform conversion */ @@ -1283,7 +1321,7 @@ void I420_A8B8G8R8( filter_t *p_filter, picture_t *p_src, picture_t *p_dest ) int i_chroma_width = (p_filter->fmt_in.video.i_x_offset + p_filter->fmt_in.video.i_visible_width) / 2; /* chroma width */ uint32_t * p_pic_start; /* beginning of the current line for copy */ /* Conversion buffer pointer */ - uint32_t * p_buffer_start = (uint32_t*)p_sys->p_buffer; + uint32_t * p_buffer_start; uint32_t * p_buffer; /* Offset array pointer */ @@ -1308,6 +1346,14 @@ void I420_A8B8G8R8( filter_t *p_filter, picture_t *p_src, picture_t *p_dest ) (p_filter->fmt_out.video.i_y_offset + p_filter->fmt_out.video.i_visible_height), &b_hscale, &i_vscale, p_offset_start ); + if(b_hscale && + AllocateOrGrow(&p_sys->p_buffer, &p_sys->i_buffer_size, + p_filter->fmt_in.video.i_x_offset + + p_filter->fmt_in.video.i_visible_width, + p_sys->i_bytespp)) + return; + else p_buffer_start = (uint32_t*)p_sys->p_buffer; + /* * Perform conversion */ _______________________________________________ vlc-commits mailing list [email protected] https://mailman.videolan.org/listinfo/vlc-commits
