From 15ccaa344c4f645ae791aafecdef3d886e196127 Mon Sep 17 00:00:00 2001 From: Michael Niedermayer Date: Mon, 20 Feb 2017 13:42:33 +0100 Subject: [PATCH] avcodec/mpegaudiodec: Eliminate many undefined operations Fixes: 625/clusterfuzz-testcase-4574924406521856 Fixes: 626/clusterfuzz-testcase-4738718621499392 Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/targets/ffmpeg Signed-off-by: Michael Niedermayer --- libavcodec/dct32_template.c | 15 +++++++++++++-- libavcodec/mpegaudio.h | 12 ++++++++++++ libavcodec/mpegaudiodec_fixed.c | 4 ++-- libavcodec/mpegaudiodec_template.c | 6 +++--- libavcodec/mpegaudiodsp_template.c | 10 +++++----- 5 files changed, 35 insertions(+), 12 deletions(-) diff --git a/libavcodec/dct32_template.c b/libavcodec/dct32_template.c index c70396e5f5..bb3f1553ac 100644 --- a/libavcodec/dct32_template.c +++ b/libavcodec/dct32_template.c @@ -21,17 +21,28 @@ #include "dct32.h" #include "mathops.h" +#include "libavutil/internal.h" + +#ifdef CHECKED +#define SUINT int +#define SUINT32 int32_t +#else +#define SUINT unsigned +#define SUINT32 uint32_t +#endif #if DCT32_FLOAT # define dct32 ff_dct32_float # define FIXHR(x) ((float)(x)) # define MULH3(x, y, s) ((s)*(y)*(x)) # define INTFLOAT float +# define SUINTFLOAT float #else # define dct32 ff_dct32_fixed # define FIXHR(a) ((int)((a) * (1LL<<32) + 0.5)) # define MULH3(x, y, s) MULH((s)*(x), y) # define INTFLOAT int +# define SUINTFLOAT SUINT #endif @@ -114,9 +125,9 @@ /* DCT32 without 1/sqrt(2) coef zero scaling. */ void dct32(INTFLOAT *out, const INTFLOAT *tab) { - INTFLOAT tmp0, tmp1; + SUINTFLOAT tmp0, tmp1; - INTFLOAT val0 , val1 , val2 , val3 , val4 , val5 , val6 , val7 , + SUINTFLOAT val0 , val1 , val2 , val3 , val4 , val5 , val6 , val7 , val8 , val9 , val10, val11, val12, val13, val14, val15, val16, val17, val18, val19, val20, val21, val22, val23, val24, val25, val26, val27, val28, val29, val30, val31; diff --git a/libavcodec/mpegaudio.h b/libavcodec/mpegaudio.h index 1591a170d7..fcded7ae3e 100644 --- a/libavcodec/mpegaudio.h +++ b/libavcodec/mpegaudio.h @@ -31,6 +31,7 @@ #endif #include +#include "libavutil/internal.h" /* max frame size, in samples */ #define MPA_FRAME_SIZE 1152 @@ -58,16 +59,27 @@ #define FIX(a) ((int)((a) * FRAC_ONE)) +#ifdef CHECKED +#define SUINT int +#define SUINT32 int32_t +#else +#define SUINT unsigned +#define SUINT32 uint32_t +#endif + #if USE_FLOATS # define INTFLOAT float +# define SUINTFLOAT float typedef float MPA_INT; typedef float OUT_INT; #elif FRAC_BITS <= 15 # define INTFLOAT int +# define SUINTFLOAT SUINT typedef int16_t MPA_INT; typedef int16_t OUT_INT; #else # define INTFLOAT int +# define SUINTFLOAT SUINT typedef int32_t MPA_INT; typedef int16_t OUT_INT; #endif diff --git a/libavcodec/mpegaudiodec_fixed.c b/libavcodec/mpegaudiodec_fixed.c index 9421ffbe94..ad7ceb20b6 100644 --- a/libavcodec/mpegaudiodec_fixed.c +++ b/libavcodec/mpegaudiodec_fixed.c @@ -25,13 +25,13 @@ #include "mpegaudio.h" -#define SHR(a,b) ((a)>>(b)) +#define SHR(a,b) (((int)(a))>>(b)) /* WARNING: only correct for positive numbers */ #define FIXR_OLD(a) ((int)((a) * FRAC_ONE + 0.5)) #define FIXR(a) ((int)((a) * FRAC_ONE + 0.5)) #define FIXHR(a) ((int)((a) * (1LL<<32) + 0.5)) #define MULH3(x, y, s) MULH((s)*(x), y) -#define MULLx(x, y, s) MULL(x,y,s) +#define MULLx(x, y, s) MULL((int)(x),(y),s) #define RENAME(a) a ## _fixed #define OUT_FMT AV_SAMPLE_FMT_S16 #define OUT_FMT_P AV_SAMPLE_FMT_S16P diff --git a/libavcodec/mpegaudiodec_template.c b/libavcodec/mpegaudiodec_template.c index 53c09edced..2fc01560fa 100644 --- a/libavcodec/mpegaudiodec_template.c +++ b/libavcodec/mpegaudiodec_template.c @@ -1182,9 +1182,9 @@ found2: } while (0) #else #define AA(j) do { \ - int tmp0 = ptr[-1-j]; \ - int tmp1 = ptr[ j]; \ - int tmp2 = MULH(tmp0 + tmp1, csa_table[j][0]); \ + SUINT tmp0 = ptr[-1-j]; \ + SUINT tmp1 = ptr[ j]; \ + SUINT tmp2 = MULH(tmp0 + tmp1, csa_table[j][0]); \ ptr[-1-j] = 4 * (tmp2 - MULH(tmp1, csa_table[j][2])); \ ptr[ j] = 4 * (tmp2 + MULH(tmp0, csa_table[j][3])); \ } while (0) diff --git a/libavcodec/mpegaudiodsp_template.c b/libavcodec/mpegaudiodsp_template.c index 62454ca306..04b8be2f05 100644 --- a/libavcodec/mpegaudiodsp_template.c +++ b/libavcodec/mpegaudiodsp_template.c @@ -63,8 +63,8 @@ static inline int round_sample(int64_t *sum) # define MACS(rt, ra, rb) MAC64(rt, ra, rb) # define MLSS(rt, ra, rb) MLS64(rt, ra, rb) # define MULH3(x, y, s) MULH((s)*(x), y) -# define MULLx(x, y, s) MULL(x,y,s) -# define SHR(a,b) ((a)>>(b)) +# define MULLx(x, y, s) MULL((int)(x),(y),s) +# define SHR(a,b) (((int)(a))>>(b)) # define FIXR(a) ((int)((a) * FRAC_ONE + 0.5)) # define FIXHR(a) ((int)((a) * (1LL<<32) + 0.5)) #endif @@ -300,11 +300,11 @@ static const INTFLOAT icos36h[9] = { }; /* using Lee like decomposition followed by hand coded 9 points DCT */ -static void imdct36(INTFLOAT *out, INTFLOAT *buf, INTFLOAT *in, INTFLOAT *win) +static void imdct36(INTFLOAT *out, INTFLOAT *buf, SUINTFLOAT *in, INTFLOAT *win) { int i, j; - INTFLOAT t0, t1, t2, t3, s0, s1, s2, s3; - INTFLOAT tmp[18], *tmp1, *in1; + SUINTFLOAT t0, t1, t2, t3, s0, s1, s2, s3; + SUINTFLOAT tmp[18], *tmp1, *in1; for (i = 17; i >= 1; i--) in[i] += in[i-1];