From b02acd029f2cf822587d0ae9086e355fe96393a3 Mon Sep 17 00:00:00 2001 From: Michael Niedermayer Date: Sun, 16 Feb 2020 20:11:52 +0100 Subject: [PATCH] swscale/output: Fix integer overflow in yuv2rgb_write_full() with out of range input Fixes: signed integer overflow: 1169365504 + 981452800 cannot be represented in type 'int' Fixes: ticket8293 Found-by: Suhwan Signed-off-by: Michael Niedermayer (cherry picked from commit e057e83a4ff4c0eeeb78dffe58e21af951c056b6) Signed-off-by: Michael Niedermayer --- libswscale/output.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/libswscale/output.c b/libswscale/output.c index 1d1df951cf..4c282a2ede 100644 --- a/libswscale/output.c +++ b/libswscale/output.c @@ -1634,9 +1634,9 @@ static av_always_inline void yuv2rgb_write_full(SwsContext *c, Y -= c->yuv2rgb_y_offset; Y *= c->yuv2rgb_y_coeff; Y += 1 << 21; - R = Y + V*c->yuv2rgb_v2r_coeff; - G = Y + V*c->yuv2rgb_v2g_coeff + U*c->yuv2rgb_u2g_coeff; - B = Y + U*c->yuv2rgb_u2b_coeff; + R = (unsigned)Y + V*c->yuv2rgb_v2r_coeff; + G = (unsigned)Y + V*c->yuv2rgb_v2g_coeff + U*c->yuv2rgb_u2g_coeff; + B = (unsigned)Y + U*c->yuv2rgb_u2b_coeff; if ((R | G | B) & 0xC0000000) { R = av_clip_uintp2(R, 30); G = av_clip_uintp2(G, 30);