videolan
/
ffmpeg
mirror of https://git.videolan.org/git/ffmpeg.git
1
Fork 0
Code Releases Activity

swscale: prevent overflow in coefficient calculation.

This commit is contained in:
Ronald S. Bultje 2011-10-19 19:56:56 -07:00
parent 65d3176aaf
commit 8305041e13
1 changed files with 11 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
libswscale/utils.c
View File

@ -271,19 +271,20 @@ static int initFilter(int16_t **outFilter, int16_t **filterPos, int *outFilterSi
floatd= d * (1.0/(1<<30));
if (flags & SWS_BICUBIC) {
#define SQRT_INT64_MAX 0xb504f333
int64_t B= (param[0] != SWS_PARAM_DEFAULT ? param[0] : 0) * (1<<24);
int64_t C= (param[1] != SWS_PARAM_DEFAULT ? param[1] : 0.6) * (1<<24);
int64_t dd = d > SQRT_INT64_MAX ? ((d >> 1) * d) >> 29 : (d * d) >> 30;
int64_t ddd = d > SQRT_INT64_MAX || dd > SQRT_INT64_MAX ?
((dd >> 2) * d) >> 28 : (dd * d) >> 30;
if (d < 1LL<<30)
coeff = (12*(1<<24)-9*B-6*C)*ddd + (-18*(1<<24)+12*B+6*C)*dd + (6*(1<<24)-2*B)*(1<<30);
else if (d < 1LL<<31)
coeff = (-B-6*C)*ddd + (6*B+30*C)*dd + (-12*B-48*C)*d + (8*B+24*C)*(1<<30);
else
coeff=0.0;
if (d >= 1LL<<31) {
coeff = 0.0;
} else {
int64_t dd = (d * d) >> 30;
int64_t ddd = (dd * d) >> 30;
if (d < 1LL<<30)
coeff = (12*(1<<24)-9*B-6*C)*ddd + (-18*(1<<24)+12*B+6*C)*dd + (6*(1<<24)-2*B)*(1<<30);
else
coeff = (-B-6*C)*ddd + (6*B+30*C)*dd + (-12*B-48*C)*d + (8*B+24*C)*(1<<30);
}
coeff *= fone>>(30+24);
}
/* else if (flags & SWS_X) {