avcodec/dnxhddec: check and propagate function return value

Similar to CVE-2013-0868, here return value check for 'init_vlc' is needed.
crafted DNxHD data can cause unspecified impact.

Reviewed-by: Paul B Mahol <onemda@gmail.com>
Signed-off-by: James Almer <jamrial@gmail.com>
(cherry picked from commit 7150f95756)
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
This commit is contained in:
maryam ebr 2021-08-03 01:05:47 -04:00 committed by Michael Niedermayer
parent 764de1f6d8
commit c7b205dedd

View File

@ -78,6 +78,7 @@ static av_cold int dnxhd_decode_init(AVCodecContext *avctx)
static int dnxhd_init_vlc(DNXHDContext *ctx, uint32_t cid) static int dnxhd_init_vlc(DNXHDContext *ctx, uint32_t cid)
{ {
int ret;
if (cid != ctx->cid) { if (cid != ctx->cid) {
int index; int index;
@ -96,21 +97,28 @@ static int dnxhd_init_vlc(DNXHDContext *ctx, uint32_t cid)
ff_free_vlc(&ctx->dc_vlc); ff_free_vlc(&ctx->dc_vlc);
ff_free_vlc(&ctx->run_vlc); ff_free_vlc(&ctx->run_vlc);
init_vlc(&ctx->ac_vlc, DNXHD_VLC_BITS, 257, if ((ret = init_vlc(&ctx->ac_vlc, DNXHD_VLC_BITS, 257,
ctx->cid_table->ac_bits, 1, 1, ctx->cid_table->ac_bits, 1, 1,
ctx->cid_table->ac_codes, 2, 2, 0); ctx->cid_table->ac_codes, 2, 2, 0)) < 0)
init_vlc(&ctx->dc_vlc, DNXHD_DC_VLC_BITS, ctx->bit_depth + 4, goto out;
if ((ret = init_vlc(&ctx->dc_vlc, DNXHD_DC_VLC_BITS, ctx->bit_depth + 4,
ctx->cid_table->dc_bits, 1, 1, ctx->cid_table->dc_bits, 1, 1,
ctx->cid_table->dc_codes, 1, 1, 0); ctx->cid_table->dc_codes, 1, 1, 0)) < 0)
init_vlc(&ctx->run_vlc, DNXHD_VLC_BITS, 62, goto out;
if ((ret = init_vlc(&ctx->run_vlc, DNXHD_VLC_BITS, 62,
ctx->cid_table->run_bits, 1, 1, ctx->cid_table->run_bits, 1, 1,
ctx->cid_table->run_codes, 2, 2, 0); ctx->cid_table->run_codes, 2, 2, 0)) < 0)
goto out;
ff_init_scantable(ctx->idsp.idct_permutation, &ctx->scantable, ff_init_scantable(ctx->idsp.idct_permutation, &ctx->scantable,
ff_zigzag_direct); ff_zigzag_direct);
ctx->cid = cid; ctx->cid = cid;
} }
return 0; ret = 0;
out:
if (ret < 0)
av_log(ctx->avctx, AV_LOG_ERROR, "init_vlc failed\n");
return ret;
} }
static int dnxhd_decode_header(DNXHDContext *ctx, AVFrame *frame, static int dnxhd_decode_header(DNXHDContext *ctx, AVFrame *frame,