Commit fe6b6d38 authored by Yaowu Xu's avatar Yaowu Xu

Use int for pixel value difference

This fixes overflow of unsigned int when the difference is negative.

Change-Id: I7a1ef2af84e59e675ec27f91de4e6d92a9c1d90c
parent a3028ddf
......@@ -368,10 +368,10 @@ class Trans16x16TestBase {
for (int j = 0; j < kNumCoeffs; ++j) {
#if CONFIG_AOM_HIGHBITDEPTH
const uint32_t diff =
const int32_t diff =
bit_depth_ == AOM_BITS_8 ? dst[j] - src[j] : dst16[j] - src16[j];
#else
const uint32_t diff = dst[j] - src[j];
const int32_t diff = dst[j] - src[j];
#endif
const uint32_t error = diff * diff;
if (max_error < error) max_error = error;
......
......@@ -154,10 +154,10 @@ TEST_P(Trans32x32Test, AccuracyCheck) {
for (int j = 0; j < kNumCoeffs; ++j) {
#if CONFIG_AOM_HIGHBITDEPTH
const uint32_t diff =
const int32_t diff =
bit_depth_ == AOM_BITS_8 ? dst[j] - src[j] : dst16[j] - src16[j];
#else
const uint32_t diff = dst[j] - src[j];
const int32_t diff = dst[j] - src[j];
#endif
const uint32_t error = diff * diff;
if (max_error < error) max_error = error;
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment