Module: libav Branch: release/12 Commit: a4eba5d151997cef81b4fd2dbc3c0ff5d88b491b
Author: Andreas Cadhalpun <[email protected]> Committer: Sean McGovern <[email protected]> Date: Sat Nov 19 14:21:11 2016 +0100 smacker: limit recursion depth of smacker_decode_bigtree This fixes segmentation faults due to stack-overflow caused by too deep recursion. Reviewed-by: Michael Niedermayer <[email protected]> Signed-off-by: Andreas Cadhalpun <[email protected]> Signed-off-by: Sean McGovern <[email protected]> (cherry picked from commit 0ccddbad200c1d9439c5a836501917d515cddf76) Signed-off-by: Sean McGovern <[email protected]> --- libavcodec/smacker.c | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/libavcodec/smacker.c b/libavcodec/smacker.c index 3f0e6cb..09c6c8d 100644 --- a/libavcodec/smacker.c +++ b/libavcodec/smacker.c @@ -43,6 +43,7 @@ #define SMKTREE_BITS 9 #define SMK_NODE 0x80000000 #define SMKTREE_DECODE_MAX_RECURSION 32 +#define SMKTREE_DECODE_BIG_MAX_RECURSION 500 typedef struct SmackVContext { AVCodecContext *avctx; @@ -130,8 +131,15 @@ static int smacker_decode_tree(GetBitContext *gb, HuffContext *hc, uint32_t pref /** * Decode header tree */ -static int smacker_decode_bigtree(GetBitContext *gb, HuffContext *hc, DBCtx *ctx) +static int smacker_decode_bigtree(GetBitContext *gb, HuffContext *hc, + DBCtx *ctx, int length) { + // Larger length can cause segmentation faults due to too deep recursion. + if (length > SMKTREE_DECODE_BIG_MAX_RECURSION) { + av_log(NULL, AV_LOG_ERROR, "Maximum bigtree recursion level exceeded.\n"); + return AVERROR_INVALIDDATA; + } + if (hc->current + 1 >= hc->length) { av_log(NULL, AV_LOG_ERROR, "Tree size exceeded!\n"); return -1; @@ -160,12 +168,12 @@ static int smacker_decode_bigtree(GetBitContext *gb, HuffContext *hc, DBCtx *ctx int r = 0, r_new, t; t = hc->current++; - r = smacker_decode_bigtree(gb, hc, ctx); + r = smacker_decode_bigtree(gb, hc, ctx, length + 1); if(r < 0) return r; hc->values[t] = SMK_NODE | r; r++; - r_new = smacker_decode_bigtree(gb, hc, ctx); + r_new = smacker_decode_bigtree(gb, hc, ctx, length + 1); if (r_new < 0) return r_new; return r + r_new; @@ -265,7 +273,7 @@ static int smacker_decode_header_tree(SmackVContext *smk, GetBitContext *gb, int goto error; } - if (smacker_decode_bigtree(gb, &huff, &ctx) < 0) + if (smacker_decode_bigtree(gb, &huff, &ctx, 0) < 0) err = -1; skip_bits1(gb); if(ctx.last[0] == -1) ctx.last[0] = huff.current++; _______________________________________________ libav-commits mailing list [email protected] https://lists.libav.org/mailman/listinfo/libav-commits
