Commit 59b969da authored by Yaowu Xu's avatar Yaowu Xu

fdct4x4_test: fix unsigned overflow

The difference between src and dst will be signed, the error will be
unsigned. The change quiets -fsanitize=integer:
    unsigned integer overflow: 4294967295 * 4294967295

Change-Id: I131cefcc9583ee8a5b98eb5182fd30e9c7237ea0
parent 8db9faef
......@@ -533,10 +533,10 @@ class Trans16x16TestBase {
for (int j = 0; j < kNumCoeffs; ++j) {
#if CONFIG_AOM_HIGHBITDEPTH
const uint32_t diff =
const int diff =
bit_depth_ == AOM_BITS_8 ? dst[j] - src[j] : dst16[j] - src16[j];
#else
const uint32_t diff = dst[j] - src[j];
const int diff = dst[j] - src[j];
#endif // CONFIG_AOM_HIGHBITDEPTH
const uint32_t error = diff * diff;
EXPECT_GE(1u, error) << "Error: 16x16 IDCT has error " << error
......@@ -589,10 +589,10 @@ class Trans16x16TestBase {
for (int j = 0; j < kNumCoeffs; ++j) {
#if CONFIG_AOM_HIGHBITDEPTH
const uint32_t diff =
const int diff =
bit_depth_ == AOM_BITS_8 ? dst[j] - ref[j] : dst16[j] - ref16[j];
#else
const uint32_t diff = dst[j] - ref[j];
const int diff = dst[j] - ref[j];
#endif // CONFIG_AOM_HIGHBITDEPTH
const uint32_t error = diff * diff;
EXPECT_EQ(0u, error) << "Error: 16x16 IDCT Comparison has error "
......
......@@ -90,11 +90,11 @@ class TransformTestBase {
for (int j = 0; j < num_coeffs_; ++j) {
#if CONFIG_AOM_HIGHBITDEPTH
const uint32_t diff =
const int diff =
bit_depth_ == AOM_BITS_8 ? dst[j] - src[j] : dst16[j] - src16[j];
#else
ASSERT_EQ(AOM_BITS_8, bit_depth_);
const uint32_t diff = dst[j] - src[j];
const int diff = dst[j] - src[j];
#endif
const uint32_t error = diff * diff;
if (max_error < error) max_error = error;
......@@ -309,10 +309,10 @@ class TransformTestBase {
for (int j = 0; j < num_coeffs_; ++j) {
#if CONFIG_AOM_HIGHBITDEPTH
const uint32_t diff =
const int diff =
bit_depth_ == AOM_BITS_8 ? dst[j] - src[j] : dst16[j] - src16[j];
#else
const uint32_t diff = dst[j] - src[j];
const int diff = dst[j] - src[j];
#endif
const uint32_t error = diff * diff;
EXPECT_GE(static_cast<uint32_t>(limit), error)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment