From e6d5edb03c846c38f38a96a651f93b305d95319f Mon Sep 17 00:00:00 2001 From: Michael Niedermayer Date: Tue, 18 Jun 2024 15:48:24 +0200 Subject: [PATCH] avcodec/jfdctint_template: Fewer integer anomalies Fixes: signed integer overflow: 105788 * -20995 cannot be represented in type 'int' Fixes: signed integer overflow: 923211729 + 2073948236 cannot be represented in type 'int' Fixes: signed integer overflow: 1281179284 + 2073948236 cannot be represented in type 'int' Fixes: 68975/clusterfuzz-testcase-minimized-ffmpeg_AV_CODEC_ID_PRORES_fuzzer-6266769177116672 Fixes: 68997/clusterfuzz-testcase-minimized-ffmpeg_AV_CODEC_ID_PRORES_KS_fuzzer-6284237161431040 Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg Signed-off-by: Michael Niedermayer (cherry picked from commit 228f255b5d9b839149cd53f0537ce76b068228ae) Signed-off-by: Michael Niedermayer --- libavcodec/jfdctint_template.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/libavcodec/jfdctint_template.c b/libavcodec/jfdctint_template.c index 67fb77b5e1..2f4e28b3a3 100644 --- a/libavcodec/jfdctint_template.c +++ b/libavcodec/jfdctint_template.c @@ -69,7 +69,7 @@ #define GLOBAL(x) x #define RIGHT_SHIFT(x, n) ((x) >> (n)) #define MULTIPLY16C16(var,const) ((var)*(const)) -#define DESCALE(x,n) RIGHT_SHIFT((x) + (1 << ((n) - 1)), n) +#define DESCALE(x,n) RIGHT_SHIFT((int)(x) + (1 << ((n) - 1)), n) /* @@ -175,7 +175,7 @@ #if BITS_IN_JSAMPLE == 8 && CONST_BITS<=13 && PASS1_BITS<=2 #define MULTIPLY(var,const) MULTIPLY16C16(var,const) #else -#define MULTIPLY(var,const) ((var) * (const)) +#define MULTIPLY(var,const) (int)((var) * (unsigned)(const)) #endif @@ -261,7 +261,7 @@ FUNC(ff_jpeg_fdct_islow)(int16_t *data) { int tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7; int tmp10, tmp11, tmp12, tmp13; - int z1, z2, z3, z4, z5; + unsigned z1, z2, z3, z4, z5; int16_t *dataptr; int ctr;