Commit 98473443 authored by James Zern's avatar James Zern

Fix variance (signed integer) overflow

In the variance calculations the difference is summed and later squared.
When the sum exceeds sqrt(2^31) the value is treated as a negative when
it is shifted which gives incorrect results.

To fix this we force the multiplication to be unsigned.

The alternative fix is to shift sum down by 4 before multiplying.
However that will reduce precision.

For 16x16 blocks the maximum sum is 65280 and sqrt(2^31) is 46340 (and
change).

This change is based on:
16982342 Missed some variance casts
fea3556e Fix variance overflow

Change-Id: I2c61856cca9db54b9b81de83b4505ea81a050a0f
parent a879b4e6
......@@ -6,5 +6,6 @@ LIBVPX_TEST_SRCS-yes += fdct4x4_test.cc
LIBVPX_TEST_SRCS-yes += fdct8x8_test.cc
LIBVPX_TEST_SRCS-yes += idct8x8_test.cc
LIBVPX_TEST_SRCS-yes += test_libvpx.cc
LIBVPX_TEST_SRCS-yes += variance_test.cc
LIBVPX_TEST_DATA-yes += hantro_collage_w352h288.yuv
/*
* Copyright (c) 2012 The WebM project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include <stdlib.h>
#include <new>
#include "third_party/googletest/src/include/gtest/gtest.h"
#include "vpx_config.h"
extern "C" {
#include "vp9/encoder/variance.h"
#include "vpx/vpx_integer.h"
#include "vpx_rtcd.h"
}
namespace {
using ::std::tr1::get;
using ::std::tr1::make_tuple;
using ::std::tr1::tuple;
class VP9VarianceTest :
public ::testing::TestWithParam<tuple<int, int, vp9_variance_fn_t> > {
public:
virtual void SetUp() {
const tuple<int, int, vp9_variance_fn_t>& params = GetParam();
width_ = get<0>(params);
height_ = get<1>(params);
variance_ = get<2>(params);
block_size_ = width_ * height_;
src_ = new uint8_t[block_size_];
ref_ = new uint8_t[block_size_];
ASSERT_TRUE(src_ != NULL);
ASSERT_TRUE(ref_ != NULL);
}
virtual void TearDown() {
delete[] src_;
delete[] ref_;
}
protected:
uint8_t* src_;
uint8_t* ref_;
int width_;
int height_;
int block_size_;
vp9_variance_fn_t variance_;
};
TEST_P(VP9VarianceTest, Zero) {
for (int i = 0; i <= 255; ++i) {
memset(src_, i, block_size_);
for (int j = 0; j <= 255; ++j) {
memset(ref_, j, block_size_);
unsigned int sse;
const unsigned int var = variance_(src_, width_, ref_, width_, &sse);
EXPECT_EQ(0u, var) << "src values: " << i << "ref values: " << j;
}
}
}
TEST_P(VP9VarianceTest, OneQuarter) {
memset(src_, 255, block_size_);
const int half = block_size_ / 2;
memset(ref_, 255, half);
memset(ref_ + half, 0, half);
unsigned int sse;
const unsigned int var = variance_(src_, width_, ref_, width_, &sse);
const unsigned int expected = block_size_ * 255 * 255 / 4;
EXPECT_EQ(expected, var);
}
const vp9_variance_fn_t variance4x4_c = vp9_variance4x4_c;
const vp9_variance_fn_t variance8x8_c = vp9_variance8x8_c;
const vp9_variance_fn_t variance8x16_c = vp9_variance8x16_c;
const vp9_variance_fn_t variance16x8_c = vp9_variance16x8_c;
const vp9_variance_fn_t variance16x16_c = vp9_variance16x16_c;
INSTANTIATE_TEST_CASE_P(
C, VP9VarianceTest,
::testing::Values(make_tuple(4, 4, variance4x4_c),
make_tuple(8, 8, variance8x8_c),
make_tuple(8, 16, variance8x16_c),
make_tuple(16, 8, variance16x8_c),
make_tuple(16, 16, variance16x16_c)));
#if HAVE_MMX
const vp9_variance_fn_t variance4x4_mmx = vp9_variance4x4_mmx;
const vp9_variance_fn_t variance8x8_mmx = vp9_variance8x8_mmx;
const vp9_variance_fn_t variance8x16_mmx = vp9_variance8x16_mmx;
const vp9_variance_fn_t variance16x8_mmx = vp9_variance16x8_mmx;
const vp9_variance_fn_t variance16x16_mmx = vp9_variance16x16_mmx;
INSTANTIATE_TEST_CASE_P(
MMX, VP9VarianceTest,
::testing::Values(make_tuple(4, 4, variance4x4_mmx),
make_tuple(8, 8, variance8x8_mmx),
make_tuple(8, 16, variance8x16_mmx),
make_tuple(16, 8, variance16x8_mmx),
make_tuple(16, 16, variance16x16_mmx)));
#endif
#if HAVE_SSE2
const vp9_variance_fn_t variance4x4_wmt = vp9_variance4x4_wmt;
const vp9_variance_fn_t variance8x8_wmt = vp9_variance8x8_wmt;
const vp9_variance_fn_t variance8x16_wmt = vp9_variance8x16_wmt;
const vp9_variance_fn_t variance16x8_wmt = vp9_variance16x8_wmt;
const vp9_variance_fn_t variance16x16_wmt = vp9_variance16x16_wmt;
INSTANTIATE_TEST_CASE_P(
SSE2, VP9VarianceTest,
::testing::Values(make_tuple(4, 4, variance4x4_wmt),
make_tuple(8, 8, variance8x8_wmt),
make_tuple(8, 16, variance8x16_wmt),
make_tuple(16, 8, variance16x8_wmt),
make_tuple(16, 16, variance16x16_wmt)));
#endif
} // namespace
......@@ -198,7 +198,7 @@ unsigned int vp9_variance2x16_c(const unsigned char *src_ptr,
variance(src_ptr, source_stride, ref_ptr, recon_stride, 2, 16, &var, &avg);
*sse = var;
return (var - ((avg * avg) >> 5));
return (var - (((unsigned int)avg * avg) >> 5));
}
unsigned int vp9_variance16x2_c(const unsigned char *src_ptr,
......@@ -211,7 +211,7 @@ unsigned int vp9_variance16x2_c(const unsigned char *src_ptr,
variance(src_ptr, source_stride, ref_ptr, recon_stride, 16, 2, &var, &avg);
*sse = var;
return (var - ((avg * avg) >> 5));
return (var - (((unsigned int)avg * avg) >> 5));
}
unsigned int vp9_sub_pixel_variance16x2_c(const unsigned char *src_ptr,
......
......@@ -37,7 +37,8 @@ unsigned int vp9_variance32x32_c(const unsigned char *src_ptr,
variance(src_ptr, source_stride, ref_ptr, recon_stride, 32, 32, &var, &avg);
*sse = var;
return (var - ((avg * avg) >> 10));
// TODO(rbultje): in extreme cases these products will rollover.
return (var - (((unsigned int)avg * avg) >> 10));
}
#endif
......@@ -51,7 +52,7 @@ unsigned int vp9_variance16x16_c(const unsigned char *src_ptr,
variance(src_ptr, source_stride, ref_ptr, recon_stride, 16, 16, &var, &avg);
*sse = var;
return (var - ((avg * avg) >> 8));
return (var - (((unsigned int)avg * avg) >> 8));
}
unsigned int vp9_variance8x16_c(const unsigned char *src_ptr,
......@@ -64,7 +65,7 @@ unsigned int vp9_variance8x16_c(const unsigned char *src_ptr,
variance(src_ptr, source_stride, ref_ptr, recon_stride, 8, 16, &var, &avg);
*sse = var;
return (var - ((avg * avg) >> 7));
return (var - (((unsigned int)avg * avg) >> 7));
}
unsigned int vp9_variance16x8_c(const unsigned char *src_ptr,
......@@ -77,7 +78,7 @@ unsigned int vp9_variance16x8_c(const unsigned char *src_ptr,
variance(src_ptr, source_stride, ref_ptr, recon_stride, 16, 8, &var, &avg);
*sse = var;
return (var - ((avg * avg) >> 7));
return (var - (((unsigned int)avg * avg) >> 7));
}
......@@ -91,7 +92,7 @@ unsigned int vp9_variance8x8_c(const unsigned char *src_ptr,
variance(src_ptr, source_stride, ref_ptr, recon_stride, 8, 8, &var, &avg);
*sse = var;
return (var - ((avg * avg) >> 6));
return (var - (((unsigned int)avg * avg) >> 6));
}
unsigned int vp9_variance4x4_c(const unsigned char *src_ptr,
......@@ -104,7 +105,7 @@ unsigned int vp9_variance4x4_c(const unsigned char *src_ptr,
variance(src_ptr, source_stride, ref_ptr, recon_stride, 4, 4, &var, &avg);
*sse = var;
return (var - ((avg * avg) >> 4));
return (var - (((unsigned int)avg * avg) >> 4));
}
......
......@@ -89,7 +89,7 @@ unsigned int vp9_variance4x4_mmx(
vp9_get4x4var_mmx(src_ptr, source_stride, ref_ptr, recon_stride, &var, &avg);
*sse = var;
return (var - ((avg * avg) >> 4));
return (var - (((unsigned int)avg * avg) >> 4));
}
......@@ -105,7 +105,7 @@ unsigned int vp9_variance8x8_mmx(
vp9_get8x8var_mmx(src_ptr, source_stride, ref_ptr, recon_stride, &var, &avg);
*sse = var;
return (var - ((avg * avg) >> 6));
return (var - (((unsigned int)avg * avg) >> 6));
}
......@@ -148,7 +148,7 @@ unsigned int vp9_variance16x16_mmx(
var = sse0 + sse1 + sse2 + sse3;
avg = sum0 + sum1 + sum2 + sum3;
*sse = var;
return (var - ((avg * avg) >> 8));
return (var - (((unsigned int)avg * avg) >> 8));
}
unsigned int vp9_variance16x8_mmx(
......@@ -166,7 +166,7 @@ unsigned int vp9_variance16x8_mmx(
var = sse0 + sse1;
avg = sum0 + sum1;
*sse = var;
return (var - ((avg * avg) >> 7));
return (var - (((unsigned int)avg * avg) >> 7));
}
......@@ -187,7 +187,7 @@ unsigned int vp9_variance8x16_mmx(
avg = sum0 + sum1;
*sse = var;
return (var - ((avg * avg) >> 7));
return (var - (((unsigned int)avg * avg) >> 7));
}
......@@ -237,7 +237,7 @@ unsigned int vp9_sub_pixel_variance4x4_mmx
&xsum, &xxsum
);
*sse = xxsum;
return (xxsum - ((xsum * xsum) >> 4));
return (xxsum - (((unsigned int)xsum * xsum) >> 4));
}
......@@ -261,7 +261,7 @@ unsigned int vp9_sub_pixel_variance8x8_mmx
&xsum, &xxsum
);
*sse = xxsum;
return (xxsum - ((xsum * xsum) >> 6));
return (xxsum - (((unsigned int)xsum * xsum) >> 6));
}
unsigned int vp9_sub_pixel_variance16x16_mmx
......@@ -296,7 +296,7 @@ unsigned int vp9_sub_pixel_variance16x16_mmx
xxsum0 += xxsum1;
*sse = xxsum0;
return (xxsum0 - ((xsum0 * xsum0) >> 8));
return (xxsum0 - (((unsigned int)xsum0 * xsum0) >> 8));
}
......@@ -347,7 +347,7 @@ unsigned int vp9_sub_pixel_variance16x8_mmx
xxsum0 += xxsum1;
*sse = xxsum0;
return (xxsum0 - ((xsum0 * xsum0) >> 7));
return (xxsum0 - (((unsigned int)xsum0 * xsum0) >> 7));
}
unsigned int vp9_sub_pixel_variance8x16_mmx
......@@ -369,7 +369,7 @@ unsigned int vp9_sub_pixel_variance8x16_mmx
&xsum, &xxsum
);
*sse = xxsum;
return (xxsum - ((xsum * xsum) >> 7));
return (xxsum - (((unsigned int)xsum * xsum) >> 7));
}
......
......@@ -150,7 +150,7 @@ unsigned int vp9_variance4x4_wmt(
vp9_get4x4var_mmx(src_ptr, source_stride, ref_ptr, recon_stride, &var, &avg);
*sse = var;
return (var - ((avg * avg) >> 4));
return (var - (((unsigned int)avg * avg) >> 4));
}
......@@ -166,7 +166,7 @@ unsigned int vp9_variance8x8_wmt
vp9_get8x8var_sse2(src_ptr, source_stride, ref_ptr, recon_stride, &var, &avg);
*sse = var;
return (var - ((avg * avg) >> 6));
return (var - (((unsigned int)avg * avg) >> 6));
}
......@@ -184,7 +184,7 @@ unsigned int vp9_variance16x16_wmt
vp9_get16x16var_sse2(src_ptr, source_stride, ref_ptr, recon_stride, &sse0, &sum0);
*sse = sse0;
return (sse0 - ((sum0 * sum0) >> 8));
return (sse0 - (((unsigned int)sum0 * sum0) >> 8));
}
unsigned int vp9_mse16x16_wmt(
const unsigned char *src_ptr,
......@@ -218,7 +218,7 @@ unsigned int vp9_variance16x8_wmt
var = sse0 + sse1;
avg = sum0 + sum1;
*sse = var;
return (var - ((avg * avg) >> 7));
return (var - (((unsigned int)avg * avg) >> 7));
}
......@@ -238,7 +238,7 @@ unsigned int vp9_variance8x16_wmt
var = sse0 + sse1;
avg = sum0 + sum1;
*sse = var;
return (var - ((avg * avg) >> 7));
return (var - (((unsigned int)avg * avg) >> 7));
}
......@@ -261,7 +261,7 @@ unsigned int vp9_sub_pixel_variance4x4_wmt
&xsum, &xxsum
);
*sse = xxsum;
return (xxsum - ((xsum * xsum) >> 4));
return (xxsum - (((unsigned int)xsum * xsum) >> 4));
}
......@@ -302,7 +302,7 @@ unsigned int vp9_sub_pixel_variance8x8_wmt
}
*sse = xxsum;
return (xxsum - ((xsum * xsum) >> 6));
return (xxsum - (((unsigned int)xsum * xsum) >> 6));
}
unsigned int vp9_sub_pixel_variance16x16_wmt
......@@ -355,7 +355,7 @@ unsigned int vp9_sub_pixel_variance16x16_wmt
}
*sse = xxsum0;
return (xxsum0 - ((xsum0 * xsum0) >> 8));
return (xxsum0 - (((unsigned int)xsum0 * xsum0) >> 8));
}
unsigned int vp9_sub_pixel_mse16x16_wmt(
......@@ -417,7 +417,7 @@ unsigned int vp9_sub_pixel_variance16x8_wmt
}
*sse = xxsum0;
return (xxsum0 - ((xsum0 * xsum0) >> 7));
return (xxsum0 - (((unsigned int)xsum0 * xsum0) >> 7));
}
unsigned int vp9_sub_pixel_variance8x16_wmt
......@@ -457,7 +457,7 @@ unsigned int vp9_sub_pixel_variance8x16_wmt
}
*sse = xxsum;
return (xxsum - ((xsum * xsum) >> 7));
return (xxsum - (((unsigned int)xsum * xsum) >> 7));
}
......@@ -476,7 +476,7 @@ unsigned int vp9_variance_halfpixvar16x16_h_wmt(
&xsum0, &xxsum0);
*sse = xxsum0;
return (xxsum0 - ((xsum0 * xsum0) >> 8));
return (xxsum0 - (((unsigned int)xsum0 * xsum0) >> 8));
}
......@@ -494,7 +494,7 @@ unsigned int vp9_variance_halfpixvar16x16_v_wmt(
&xsum0, &xxsum0);
*sse = xxsum0;
return (xxsum0 - ((xsum0 * xsum0) >> 8));
return (xxsum0 - (((unsigned int)xsum0 * xsum0) >> 8));
}
......@@ -513,5 +513,5 @@ unsigned int vp9_variance_halfpixvar16x16_hv_wmt(
&xsum0, &xxsum0);
*sse = xxsum0;
return (xxsum0 - ((xsum0 * xsum0) >> 8));
return (xxsum0 - (((unsigned int)xsum0 * xsum0) >> 8));
}
......@@ -106,7 +106,7 @@ unsigned int vp9_sub_pixel_variance16x16_ssse3
}
*sse = xxsum0;
return (xxsum0 - ((xsum0 * xsum0) >> 8));
return (xxsum0 - (((unsigned int)xsum0 * xsum0) >> 8));
}
unsigned int vp9_sub_pixel_variance16x8_ssse3
......@@ -147,5 +147,5 @@ unsigned int vp9_sub_pixel_variance16x8_ssse3
}
*sse = xxsum0;
return (xxsum0 - ((xsum0 * xsum0) >> 7));
return (xxsum0 - (((unsigned int)xsum0 * xsum0) >> 7));
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment