Commit d3fe3b2a authored by Alex Converse's avatar Alex Converse

Avoid an unsigned overflow in invert_quant

Change-Id: I16a570b2af66b6580d1cd6f8345a25f079009bf4
parent b2ccb9c1
......@@ -219,12 +219,12 @@ void vp10_regular_quantize_b_4x4(MACROBLOCK *x, int plane, int block,
static void invert_quant(int16_t *quant, int16_t *shift, int d) {
unsigned t;
int l;
int l, m;
t = d;
for (l = 0; t > 1; l++)
t >>= 1;
t = 1 + (1 << (16 + l)) / d;
*quant = (int16_t)(t - (1 << 16));
m = 1 + (1 << (16 + l)) / d;
*quant = (int16_t)(m - (1 << 16));
*shift = 1 << (16 - l);
}
......
......@@ -227,12 +227,12 @@ static void invert_quant(int improved_quant, short *quant,
if(improved_quant)
{
unsigned t;
int l;
int l, m;
t = d;
for(l = 0; t > 1; l++)
t>>=1;
t = 1 + (1<<(16+l))/d;
*quant = (short)(t - (1<<16));
m = 1 + (1<<(16+l))/d;
*quant = (short)(m - (1<<16));
*shift = l;
/* use multiplication and constant shift by 16 */
*shift = 1 << (16 - *shift);
......
......@@ -219,12 +219,12 @@ void vp9_regular_quantize_b_4x4(MACROBLOCK *x, int plane, int block,
static void invert_quant(int16_t *quant, int16_t *shift, int d) {
unsigned t;
int l;
int l, m;
t = d;
for (l = 0; t > 1; l++)
t >>= 1;
t = 1 + (1 << (16 + l)) / d;
*quant = (int16_t)(t - (1 << 16));
m = 1 + (1 << (16 + l)) / d;
*quant = (int16_t)(m - (1 << 16));
*shift = 1 << (16 - l);
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment