avcodec/aacdec_fixed: Fix various integer overflows

Fixes: 1377/clusterfuzz-testcase-minimized-5487049807233024

Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/targets/ffmpeg
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
This commit is contained in:
Michael Niedermayer
2017-05-06 22:09:59 +02:00
parent 8ba1fc2a4a
commit ad2296ab3a
3 changed files with 18 additions and 16 deletions

View File

@@ -34,8 +34,9 @@
static SoftFloat sbr_sum_square_c(int (*x)[2], int n)
{
SoftFloat ret;
int64_t accu = 0;
int i, nz, round;
uint64_t accu = 0, round;
int i, nz;
unsigned u;
for (i = 0; i < n; i += 2) {
// Larger values are inavlid and could cause overflows of accu.
@@ -49,22 +50,22 @@ static SoftFloat sbr_sum_square_c(int (*x)[2], int n)
accu += (int64_t)x[i + 1][1] * x[i + 1][1];
}
i = (int)(accu >> 32);
if (i == 0) {
u = accu >> 32;
if (u == 0) {
nz = 1;
} else {
nz = 0;
while (FFABS(i) < 0x40000000) {
i <<= 1;
nz = -1;
while (u < 0x80000000U) {
u <<= 1;
nz++;
}
nz = 32 - nz;
}
round = 1 << (nz-1);
i = (int)((accu + round) >> nz);
i >>= 1;
ret = av_int2sf(i, 15 - nz);
round = 1ULL << (nz-1);
u = ((accu + round) >> nz);
u >>= 1;
ret = av_int2sf(u, 15 - nz);
return ret;
}
@@ -107,7 +108,8 @@ static void sbr_qmf_deint_neg_c(int *v, const int *src)
static av_always_inline SoftFloat autocorr_calc(int64_t accu)
{
int nz, mant, expo, round;
int nz, mant, expo;
unsigned round;
int i = (int)(accu >> 32);
if (i == 0) {
nz = 1;
@@ -120,7 +122,7 @@ static av_always_inline SoftFloat autocorr_calc(int64_t accu)
nz = 32-nz;
}
round = 1 << (nz-1);
round = 1U << (nz-1);
mant = (int)((accu + round) >> nz);
mant = (mant + 0x40)>>7;
mant <<= 6;