swscale/utils: Limit filter shifting so as not to read from prior the array

Fixes out of array read
Fixes: asan_heap-oob_1fb2f9b_3780_cov_3984375136_usf.mkv

Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind
Signed-off-by: Michael Niedermayer <michaelni@gmx.at>
(cherry picked from commit 692b22626e)

Signed-off-by: Michael Niedermayer <michaelni@gmx.at>
This commit is contained in:
Michael Niedermayer 2015-02-05 00:12:08 +01:00
parent b844584c10
commit 813054c4bd

View File

@ -596,14 +596,15 @@ static av_cold int initFilter(int16_t **outFilter, int32_t **filterPos,
}
if ((*filterPos)[i] + filterSize > srcW) {
int shift = (*filterPos)[i] + filterSize - srcW;
int shift = (*filterPos)[i] + FFMIN(filterSize - srcW, 0);
// move filter coefficients right to compensate for filterPos
for (j = filterSize - 2; j >= 0; j--) {
int right = FFMIN(j + shift, filterSize - 1);
filter[i * filterSize + right] += filter[i * filterSize + j];
filter[i * filterSize + j] = 0;
}
(*filterPos)[i]= srcW - filterSize;
(*filterPos)[i]-= shift;
}
}