From 5e072787830aac9e5c6408ffe7f031b0a2dc2c1d Mon Sep 17 00:00:00 2001 From: Michael Niedermayer Date: Sun, 7 May 2017 14:12:04 +0200 Subject: [PATCH] avcodec/mss34dsp: Fix multiple signed integer overflow Fixes: 1387/clusterfuzz-testcase-minimized-4802757766676480 Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/targets/ffmpeg Signed-off-by: Michael Niedermayer (cherry picked from commit 464c4b86ee43b7912e6f23fd3e5ba40381b4c371) Signed-off-by: Michael Niedermayer --- libavcodec/mss34dsp.c | 44 +++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/libavcodec/mss34dsp.c b/libavcodec/mss34dsp.c index 0397add17d..4965ac514d 100644 --- a/libavcodec/mss34dsp.c +++ b/libavcodec/mss34dsp.c @@ -62,30 +62,30 @@ void ff_mss34_gen_quant_mat(uint16_t *qmat, int quality, int luma) } #define DCT_TEMPLATE(blk, step, SOP, shift) \ - const int t0 = -39409 * blk[7 * step] - 58980 * blk[1 * step]; \ - const int t1 = 39410 * blk[1 * step] - 58980 * blk[7 * step]; \ - const int t2 = -33410 * blk[5 * step] - 167963 * blk[3 * step]; \ - const int t3 = 33410 * blk[3 * step] - 167963 * blk[5 * step]; \ - const int t4 = blk[3 * step] + blk[7 * step]; \ - const int t5 = blk[1 * step] + blk[5 * step]; \ - const int t6 = 77062 * t4 + 51491 * t5; \ - const int t7 = 77062 * t5 - 51491 * t4; \ - const int t8 = 35470 * blk[2 * step] - 85623 * blk[6 * step]; \ - const int t9 = 35470 * blk[6 * step] + 85623 * blk[2 * step]; \ - const int tA = SOP(blk[0 * step] - blk[4 * step]); \ - const int tB = SOP(blk[0 * step] + blk[4 * step]); \ + const unsigned t0 =-39409U * blk[7 * step] - 58980U * blk[1 * step]; \ + const unsigned t1 = 39410U * blk[1 * step] - 58980U * blk[7 * step]; \ + const unsigned t2 =-33410U * blk[5 * step] -167963U * blk[3 * step]; \ + const unsigned t3 = 33410U * blk[3 * step] -167963U * blk[5 * step]; \ + const unsigned t4 = blk[3 * step] + blk[7 * step]; \ + const unsigned t5 = blk[1 * step] + blk[5 * step]; \ + const unsigned t6 = 77062U * t4 + 51491U * t5; \ + const unsigned t7 = 77062U * t5 - 51491U * t4; \ + const unsigned t8 = 35470U * blk[2 * step] - 85623U * blk[6 * step]; \ + const unsigned t9 = 35470U * blk[6 * step] + 85623U * blk[2 * step]; \ + const unsigned tA = SOP(blk[0 * step] - blk[4 * step]); \ + const unsigned tB = SOP(blk[0 * step] + blk[4 * step]); \ \ - blk[0 * step] = ( t1 + t6 + t9 + tB) >> shift; \ - blk[1 * step] = ( t3 + t7 + t8 + tA) >> shift; \ - blk[2 * step] = ( t2 + t6 - t8 + tA) >> shift; \ - blk[3 * step] = ( t0 + t7 - t9 + tB) >> shift; \ - blk[4 * step] = (-(t0 + t7) - t9 + tB) >> shift; \ - blk[5 * step] = (-(t2 + t6) - t8 + tA) >> shift; \ - blk[6 * step] = (-(t3 + t7) + t8 + tA) >> shift; \ - blk[7 * step] = (-(t1 + t6) + t9 + tB) >> shift; \ + blk[0 * step] = (int)( t1 + t6 + t9 + tB) >> shift; \ + blk[1 * step] = (int)( t3 + t7 + t8 + tA) >> shift; \ + blk[2 * step] = (int)( t2 + t6 - t8 + tA) >> shift; \ + blk[3 * step] = (int)( t0 + t7 - t9 + tB) >> shift; \ + blk[4 * step] = (int)(-(t0 + t7) - t9 + tB) >> shift; \ + blk[5 * step] = (int)(-(t2 + t6) - t8 + tA) >> shift; \ + blk[6 * step] = (int)(-(t3 + t7) + t8 + tA) >> shift; \ + blk[7 * step] = (int)(-(t1 + t6) + t9 + tB) >> shift; \ -#define SOP_ROW(a) (((a) << 16) + 0x2000) -#define SOP_COL(a) (((a) + 32) << 16) +#define SOP_ROW(a) (((a) * (1U << 16)) + 0x2000) +#define SOP_COL(a) (((a) + 32) * (1U << 16)) void ff_mss34_dct_put(uint8_t *dst, int stride, int *block) {