Commit 39eb44b2 authored by Yaowu Xu's avatar Yaowu Xu

Use int type for coeff difference

This fix unsigned int overflow when the difference is negative.

Change-Id: I423ca1cdee529c75d3fb7232b9e4382e658faa27
parent fe6b6d38
...@@ -444,7 +444,7 @@ class FwdTrans8x8TestBase { ...@@ -444,7 +444,7 @@ class FwdTrans8x8TestBase {
coeff_r[j] = static_cast<tran_low_t>(round(out_r[j])); coeff_r[j] = static_cast<tran_low_t>(round(out_r[j]));
for (int j = 0; j < kNumCoeffs; ++j) { for (int j = 0; j < kNumCoeffs; ++j) {
const uint32_t diff = coeff[j] - coeff_r[j]; const int32_t diff = coeff[j] - coeff_r[j];
const uint32_t error = diff * diff; const uint32_t error = diff * diff;
EXPECT_GE(9u << 2 * (bit_depth_ - 8), error) EXPECT_GE(9u << 2 * (bit_depth_ - 8), error)
<< "Error: 8x8 DCT has error " << error << " at index " << j; << "Error: 8x8 DCT has error " << error << " at index " << j;
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment