Commit d8a18186 authored by Alex Converse's avatar Alex Converse

pickmode: Fix a pair of unsigned overflows.

block_variance: This operates on 8x8s and would be safe with a int32 *
int32 to uint32 multiply, but this is potentially unsafe for 12-bit
input. Unfortunately the code already segfaults on 12-bit input:
https://bugs.chromium.org/p/webm/issues/detail?id=1223

calculate_variance: This operates on up to a 32x32 of 8x8s and can
overflow even with 8-bit input (log2((256*32*32)**2) == 36).

BUG=https://bugs.chromium.org/p/webm/issues/detail?id=1220

Change-Id: I1ca4ff6092db9a7580da371ee9a21f403fdadc40
parent bd2e7fa1
......@@ -244,7 +244,7 @@ static void block_variance(const uint8_t *src, int src_stride,
&sse8x8[k], &sum8x8[k]);
*sse += sse8x8[k];
*sum += sum8x8[k];
var8x8[k] = sse8x8[k] - (((unsigned int)sum8x8[k] * sum8x8[k]) >> 6);
var8x8[k] = sse8x8[k] - (uint32_t)(((int64_t)sum8x8[k] * sum8x8[k]) >> 6);
k++;
}
}
......@@ -265,7 +265,7 @@ static void calculate_variance(int bw, int bh, TX_SIZE tx_size,
sse_i[(i + 1) * nw + j] + sse_i[(i + 1) * nw + j + 1];
sum_o[k] = sum_i[i * nw + j] + sum_i[i * nw + j + 1] +
sum_i[(i + 1) * nw + j] + sum_i[(i + 1) * nw + j + 1];
var_o[k] = sse_o[k] - (((unsigned int)sum_o[k] * sum_o[k]) >>
var_o[k] = sse_o[k] - (uint32_t)(((int64_t)sum_o[k] * sum_o[k]) >>
(b_width_log2_lookup[unit_size] +
b_height_log2_lookup[unit_size] + 6));
k++;
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment