From 20afb96a930ce3dc003e3b19069f1340fed05a9b Mon Sep 17 00:00:00 2001 From: Paul B Mahol Date: Fri, 9 Aug 2019 11:30:19 +0200 Subject: [PATCH] avcodec/mlpenc: fix invalid checks, sample buffers are internally all in 24 depth While here also fix possible memory leaks. --- This fixes encoding for 16bit depth for me. Except for last frame, which remain not encoded. If you have still artifacts, please share input file. --- libavcodec/mlpenc.c | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/libavcodec/mlpenc.c b/libavcodec/mlpenc.c index deb171645c..078428c5d4 100644 --- a/libavcodec/mlpenc.c +++ b/libavcodec/mlpenc.c @@ -1807,13 +1807,13 @@ static int apply_filter(MLPEncodeContext *ctx, unsigned int channel) { FilterParams *fp[NUM_FILTERS] = { &ctx->cur_channel_params[channel].filter_params[FIR], &ctx->cur_channel_params[channel].filter_params[IIR], }; - int32_t *filter_state_buffer[NUM_FILTERS]; + int32_t *filter_state_buffer[NUM_FILTERS] = { NULL }; int32_t mask = MSB_MASK(ctx->cur_decoding_params->quant_step_size[channel]); int32_t *sample_buffer = ctx->sample_buffer + channel; unsigned int number_of_samples = ctx->number_of_samples; unsigned int filter_shift = fp[FIR]->shift; int filter; - int i; + int i, ret = 0; for (i = 0; i < NUM_FILTERS; i++) { unsigned int size = ctx->number_of_samples; @@ -1821,7 +1821,8 @@ static int apply_filter(MLPEncodeContext *ctx, unsigned int channel) if (!filter_state_buffer[i]) { av_log(ctx->avctx, AV_LOG_ERROR, "Not enough memory for applying filters.\n"); - return -1; + ret = -1; + goto end; } } @@ -1848,8 +1849,10 @@ static int apply_filter(MLPEncodeContext *ctx, unsigned int channel) accum >>= filter_shift; residual = sample - (accum & mask); - if (residual < SAMPLE_MIN(ctx->wordlength) || residual > SAMPLE_MAX(ctx->wordlength)) - return -1; + if (residual < SAMPLE_MIN(24) || residual > SAMPLE_MAX(24)) { + ret = -1; + goto end; + } filter_state_buffer[FIR][i] = sample; filter_state_buffer[IIR][i] = residual; @@ -1864,11 +1867,12 @@ static int apply_filter(MLPEncodeContext *ctx, unsigned int channel) sample_buffer += ctx->num_channels; } +end: for (i = 0; i < NUM_FILTERS; i++) { av_freep(&filter_state_buffer[i]); } - return 0; + return ret; } static void apply_filters(MLPEncodeContext *ctx) -- 2.22.0