Commit 39eb44b2 authored by Yaowu Xu's avatar Yaowu Xu

Use int type for coeff difference

This fix unsigned int overflow when the difference is negative.

Change-Id: I423ca1cdee529c75d3fb7232b9e4382e658faa27
parent fe6b6d38
......@@ -444,7 +444,7 @@ class FwdTrans8x8TestBase {
coeff_r[j] = static_cast<tran_low_t>(round(out_r[j]));
for (int j = 0; j < kNumCoeffs; ++j) {
const uint32_t diff = coeff[j] - coeff_r[j];
const int32_t diff = coeff[j] - coeff_r[j];
const uint32_t error = diff * diff;
EXPECT_GE(9u << 2 * (bit_depth_ - 8), error)
<< "Error: 8x8 DCT has error " << error << " at index " << j;
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment