From 8b9e9ba0c2f6d52b993cf6654fc443c98707eb24 Mon Sep 17 00:00:00 2001 From: Michael Niedermayer Date: Thu, 5 Oct 2023 18:27:28 +0200 Subject: [PATCH] avcodec/xvididct: Make c* unsigned to avoid undefined overflows Fixes: signed integer overflow: 1496950099 + 728014168 cannot be represented in type 'int' Fixes: 62667/clusterfuzz-testcase-minimized-ffmpeg_AV_CODEC_ID_MJPEGB_fuzzer-6511785170305024 Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg Signed-off-by: Michael Niedermayer (cherry picked from commit f7e5537dc1ff2f45a6e4c98091f15e60c3647cfc) Signed-off-by: Michael Niedermayer --- libavcodec/xvididct.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/libavcodec/xvididct.c b/libavcodec/xvididct.c index f09ce3d544..ced8c7235a 100644 --- a/libavcodec/xvididct.c +++ b/libavcodec/xvididct.c @@ -56,13 +56,13 @@ static const int TAB35[] = { 26722, 25172, 22654, 19266, 15137, 10426, 5315 }; static int idct_row(short *in, const int *const tab, int rnd) { - const int c1 = tab[0]; - const int c2 = tab[1]; - const int c3 = tab[2]; - const int c4 = tab[3]; - const int c5 = tab[4]; - const int c6 = tab[5]; - const int c7 = tab[6]; + const unsigned c1 = tab[0]; + const unsigned c2 = tab[1]; + const unsigned c3 = tab[2]; + const unsigned c4 = tab[3]; + const unsigned c5 = tab[4]; + const unsigned c6 = tab[5]; + const unsigned c7 = tab[6]; const int right = in[5] | in[6] | in[7]; const int left = in[1] | in[2] | in[3]; @@ -102,8 +102,8 @@ static int idct_row(short *in, const int *const tab, int rnd) return 0; } } else if (!(left | right)) { - const int a0 = (rnd + c4 * (in[0] + in[4])) >> ROW_SHIFT; - const int a1 = (rnd + c4 * (in[0] - in[4])) >> ROW_SHIFT; + const int a0 = (int)(rnd + c4 * (in[0] + in[4])) >> ROW_SHIFT; + const int a1 = (int)(rnd + c4 * (in[0] - in[4])) >> ROW_SHIFT; in[0] = a0; in[3] = a0;