Commit 98473443 authored by James Zern's avatar James Zern
Browse files

Fix variance (signed integer) overflow

In the variance calculations the difference is summed and later squared.
When the sum exceeds sqrt(2^31) the value is treated as a negative when
it is shifted which gives incorrect results.

To fix this we force the multiplication to be unsigned.

The alternative fix is to shift sum down by 4 before multiplying.
However that will reduce precision.

For 16x16 blocks the maximum sum is 65280 and sqrt(2^31) is 46340 (and
change).

This change is based on:
16982342 Missed some variance casts
fea3556e Fix variance overflow

Change-Id: I2c61856cca9db54b9b81de83b4505ea81a050a0f
parent a879b4e6
......@@ -6,5 +6,6 @@ LIBVPX_TEST_SRCS-yes += fdct4x4_test.cc
LIBVPX_TEST_SRCS-yes += fdct8x8_test.cc
LIBVPX_TEST_SRCS-yes += idct8x8_test.cc
LIBVPX_TEST_SRCS-yes += test_libvpx.cc
LIBVPX_TEST_SRCS-yes += variance_test.cc
LIBVPX_TEST_DATA-yes += hantro_collage_w352h288.yuv
/*
* Copyright (c) 2012 The WebM project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include <stdlib.h>
#include <new>
#include "third_party/googletest/src/include/gtest/gtest.h"
#include "vpx_config.h"
extern "C" {
#include "vp9/encoder/variance.h"
#include "vpx/vpx_integer.h"
#include "vpx_rtcd.h"
}
namespace {
using ::std::tr1::get;
using ::std::tr1::make_tuple;
using ::std::tr1::tuple;
class VP9VarianceTest :
public ::testing::TestWithParam<tuple<int, int, vp9_variance_fn_t> > {
public:
virtual void SetUp() {
const tuple<int, int, vp9_variance_fn_t>& params = GetParam();
width_ = get<0>(params);
height_ = get<1>(params);
variance_ = get<2>(params);
block_size_ = width_ * height_;
src_ = new uint8_t[block_size_];
ref_ = new uint8_t[block_size_];
ASSERT_TRUE(src_ != NULL);
ASSERT_TRUE(ref_ != NULL);
}
virtual void TearDown() {
delete[] src_;
delete[] ref_;
}
protected:
uint8_t* src_;
uint8_t* ref_;
int width_;
int height_;
int block_size_;
vp9_variance_fn_t variance_;
};
TEST_P(VP9VarianceTest, Zero) {
for (int i = 0; i <= 255; ++i) {
memset(src_, i, block_size_);
for (int j = 0; j <= 255; ++j) {
memset(ref_, j, block_size_);
unsigned int sse;
const unsigned int var = variance_(src_, width_, ref_, width_, &sse);
EXPECT_EQ(0u, var) << "src values: " << i << "ref values: " << j;
}
}
}
TEST_P(VP9VarianceTest, OneQuarter) {
memset(src_, 255, block_size_);
const int half = block_size_ / 2;
memset(ref_, 255, half);
memset(ref_ + half, 0, half);
unsigned int sse;
const unsigned int var = variance_(src_, width_, ref_, width_, &sse);
const unsigned int expected = block_size_ * 255 * 255 / 4;
EXPECT_EQ(expected, var);
}
const vp9_variance_fn_t variance4x4_c = vp9_variance4x4_c;
const vp9_variance_fn_t variance8x8_c = vp9_variance8x8_c;
const vp9_variance_fn_t variance8x16_c = vp9_variance8x16_c;
const vp9_variance_fn_t variance16x8_c = vp9_variance16x8_c;
const vp9_variance_fn_t variance16x16_c = vp9_variance16x16_c;
INSTANTIATE_TEST_CASE_P(
C, VP9VarianceTest,
::testing::Values(make_tuple(4, 4, variance4x4_c),
make_tuple(8, 8, variance8x8_c),
make_tuple(8, 16, variance8x16_c),
make_tuple(16, 8, variance16x8_c),
make_tuple(16, 16, variance16x16_c)));
#if HAVE_MMX
const vp9_variance_fn_t variance4x4_mmx = vp9_variance4x4_mmx;
const vp9_variance_fn_t variance8x8_mmx = vp9_variance8x8_mmx;
const vp9_variance_fn_t variance8x16_mmx = vp9_variance8x16_mmx;
const vp9_variance_fn_t variance16x8_mmx = vp9_variance16x8_mmx;
const vp9_variance_fn_t variance16x16_mmx = vp9_variance16x16_mmx;
INSTANTIATE_TEST_CASE_P(
MMX, VP9VarianceTest,
::testing::Values(make_tuple(4, 4, variance4x4_mmx),
make_tuple(8, 8, variance8x8_mmx),
make_tuple(8, 16, variance8x16_mmx),
make_tuple(16, 8, variance16x8_mmx),
make_tuple(16, 16, variance16x16_mmx)));
#endif
#if HAVE_SSE2
const vp9_variance_fn_t variance4x4_wmt = vp9_variance4x4_wmt;
const vp9_variance_fn_t variance8x8_wmt = vp9_variance8x8_wmt;
const vp9_variance_fn_t variance8x16_wmt = vp9_variance8x16_wmt;
const vp9_variance_fn_t variance16x8_wmt = vp9_variance16x8_wmt;
const vp9_variance_fn_t variance16x16_wmt = vp9_variance16x16_wmt;
INSTANTIATE_TEST_CASE_P(
SSE2, VP9VarianceTest,
::testing::Values(make_tuple(4, 4, variance4x4_wmt),
make_tuple(8, 8, variance8x8_wmt),
make_tuple(8, 16, variance8x16_wmt),
make_tuple(16, 8, variance16x8_wmt),
make_tuple(16, 16, variance16x16_wmt)));
#endif
} // namespace
......@@ -198,7 +198,7 @@ unsigned int vp9_variance2x16_c(const unsigned char *src_ptr,
variance(src_ptr, source_stride, ref_ptr, recon_stride, 2, 16, &var, &avg);
*sse = var;
return (var - ((avg * avg) >> 5));
return (var - (((unsigned int)avg * avg) >> 5));
}
unsigned int vp9_variance16x2_c(const unsigned char *src_ptr,
......@@ -211,7 +211,7 @@ unsigned int vp9_variance16x2_c(const unsigned char *src_ptr,
variance(src_ptr, source_stride, ref_ptr, recon_stride, 16, 2, &var, &avg);
*sse = var;
return (var - ((avg * avg) >> 5));
return (var - (((unsigned int)avg * avg) >> 5));
}
unsigned int vp9_sub_pixel_variance16x2_c(const unsigned char *src_ptr,
......
......@@ -37,7 +37,8 @@ unsigned int vp9_variance32x32_c(const unsigned char *src_ptr,
variance(src_ptr, source_stride, ref_ptr, recon_stride, 32, 32, &var, &avg);
*sse = var;
return (var - ((avg * avg) >> 10));
// TODO(rbultje): in extreme cases these products will rollover.
return (var - (((unsigned int)avg * avg) >> 10));
}
#endif
......@@ -51,7 +52,7 @@ unsigned int vp9_variance16x16_c(const unsigned char *src_ptr,
variance(src_ptr, source_stride, ref_ptr, recon_stride, 16, 16, &var, &avg);
*sse = var;
return (var - ((avg * avg) >> 8));
return (var - (((unsigned int)avg * avg) >> 8));
}
unsigned int vp9_variance8x16_c(const unsigned char *src_ptr,
......@@ -64,7 +65,7 @@ unsigned int vp9_variance8x16_c(const unsigned char *src_ptr,
variance(src_ptr, source_stride, ref_ptr, recon_stride, 8, 16, &var, &avg);
*sse = var;
return (var - ((avg * avg) >> 7));
return (var - (((unsigned int)avg * avg) >> 7));
}
unsigned int vp9_variance16x8_c(const unsigned char *src_ptr,
......@@ -77,7 +78,7 @@ unsigned int vp9_variance16x8_c(const unsigned char *src_ptr,
variance(src_ptr, source_stride, ref_ptr, recon_stride, 16, 8, &var, &avg);
*sse = var;
return (var - ((avg * avg) >> 7));
return (var - (((unsigned int)avg * avg) >> 7));
}
......@@ -91,7 +92,7 @@ unsigned int vp9_variance8x8_c(const unsigned char *src_ptr,
variance(src_ptr, source_stride, ref_ptr, recon_stride, 8, 8, &var, &avg);
*sse = var;
return (var - ((avg * avg) >> 6));
return (var - (((unsigned int)avg * avg) >> 6));
}
unsigned int vp9_variance4x4_c(const unsigned char *src_ptr,
......@@ -104,7 +105,7 @@ unsigned int vp9_variance4x4_c(const unsigned char *src_ptr,
variance(src_ptr, source_stride, ref_ptr, recon_stride, 4, 4, &var, &avg);
*sse = var;
return (var - ((avg * avg) >> 4));
return (var - (((unsigned int)avg * avg) >> 4));
}
......
......@@ -89,7 +89,7 @@ unsigned int vp9_variance4x4_mmx(
vp9_get4x4var_mmx(src_ptr, source_stride, ref_ptr, recon_stride, &var, &avg);
*sse = var;
return (var - ((avg * avg) >> 4));
return (var - (((unsigned int)avg * avg) >> 4));
}
......@@ -105,7 +105,7 @@ unsigned int vp9_variance8x8_mmx(
vp9_get8x8var_mmx(src_ptr, source_stride, ref_ptr, recon_stride, &var, &avg);
*sse = var;
return (var - ((avg * avg) >> 6));
return (var - (((unsigned int)avg * avg) >> 6));
}
......@@ -148,7 +148,7 @@ unsigned int vp9_variance16x16_mmx(
var = sse0 + sse1 + sse2 + sse3;
avg = sum0 + sum1 + sum2 + sum3;
*sse = var;
return (var - ((avg * avg) >> 8));
return (var - (((unsigned int)avg * avg) >> 8));
}
unsigned int vp9_variance16x8_mmx(
......@@ -166,7 +166,7 @@ unsigned int vp9_variance16x8_mmx(
var = sse0 + sse1;
avg = sum0 + sum1;
*sse = var;
return (var - ((avg * avg) >> 7));
return (var - (((unsigned int)avg * avg) >> 7));
}
......@@ -187,7 +187,7 @@ unsigned int vp9_variance8x16_mmx(
avg = sum0 + sum1;
*sse = var;
return (var - ((avg * avg) >> 7));
return (var - (((unsigned int)avg * avg) >> 7));
}
......@@ -237,7 +237,7 @@ unsigned int vp9_sub_pixel_variance4x4_mmx
&xsum, &xxsum
);
*sse = xxsum;
return (xxsum - ((xsum * xsum) >> 4));
return (xxsum - (((unsigned int)xsum * xsum) >> 4));
}
......@@ -261,7 +261,7 @@ unsigned int vp9_sub_pixel_variance8x8_mmx
&xsum, &xxsum
);
*sse = xxsum;
return (xxsum - ((xsum * xsum) >> 6));
return (xxsum - (((unsigned int)xsum * xsum) >> 6));
}
unsigned int vp9_sub_pixel_variance16x16_mmx
......@@ -296,7 +296,7 @@ unsigned int vp9_sub_pixel_variance16x16_mmx
xxsum0 += xxsum1;
*sse = xxsum0;
return (xxsum0 - ((xsum0 * xsum0) >> 8));
return (xxsum0 - (((unsigned int)xsum0 * xsum0) >> 8));
}
......@@ -347,7 +347,7 @@ unsigned int vp9_sub_pixel_variance16x8_mmx
xxsum0 += xxsum1;
*sse = xxsum0;
return (xxsum0 - ((xsum0 * xsum0) >> 7));
return (xxsum0 - (((unsigned int)xsum0 * xsum0) >> 7));
}
unsigned int vp9_sub_pixel_variance8x16_mmx
......@@ -369,7 +369,7 @@ unsigned int vp9_sub_pixel_variance8x16_mmx
&xsum, &xxsum
);
*sse = xxsum;
return (xxsum - ((xsum * xsum) >> 7));
return (xxsum - (((unsigned int)xsum * xsum) >> 7));
}
......
......@@ -150,7 +150,7 @@ unsigned int vp9_variance4x4_wmt(
vp9_get4x4var_mmx(src_ptr, source_stride, ref_ptr, recon_stride, &var, &avg);
*sse = var;
return (var - ((avg * avg) >> 4));
return (var - (((unsigned int)avg * avg) >> 4));
}
......@@ -166,7 +166,7 @@ unsigned int vp9_variance8x8_wmt
vp9_get8x8var_sse2(src_ptr, source_stride, ref_ptr, recon_stride, &var, &avg);
*sse = var;
return (var - ((avg * avg) >> 6));
return (var - (((unsigned int)avg * avg) >> 6));
}
......@@ -184,7 +184,7 @@ unsigned int vp9_variance16x16_wmt
vp9_get16x16var_sse2(src_ptr, source_stride, ref_ptr, recon_stride, &sse0, &sum0);
*sse = sse0;
return (sse0 - ((sum0 * sum0) >> 8));
return (sse0 - (((unsigned int)sum0 * sum0) >> 8));
}
unsigned int vp9_mse16x16_wmt(
const unsigned char *src_ptr,
......@@ -218,7 +218,7 @@ unsigned int vp9_variance16x8_wmt
var = sse0 + sse1;
avg = sum0 + sum1;
*sse = var;
return (var - ((avg * avg) >> 7));
return (var - (((unsigned int)avg * avg) >> 7));
}
......@@ -238,7 +238,7 @@ unsigned int vp9_variance8x16_wmt
var = sse0 + sse1;
avg = sum0 + sum1;
*sse = var;
return (var - ((avg * avg) >> 7));
return (var - (((unsigned int)avg * avg) >> 7));
}
......@@ -261,7 +261,7 @@ unsigned int vp9_sub_pixel_variance4x4_wmt
&xsum, &xxsum
);
*sse = xxsum;
return (xxsum - ((xsum * xsum) >> 4));
return (xxsum - (((unsigned int)xsum * xsum) >> 4));
}
......@@ -302,7 +302,7 @@ unsigned int vp9_sub_pixel_variance8x8_wmt
}
*sse = xxsum;
return (xxsum - ((xsum * xsum) >> 6));
return (xxsum - (((unsigned int)xsum * xsum) >> 6));
}
unsigned int vp9_sub_pixel_variance16x16_wmt
......@@ -355,7 +355,7 @@ unsigned int vp9_sub_pixel_variance16x16_wmt
}
*sse = xxsum0;
return (xxsum0 - ((xsum0 * xsum0) >> 8));
return (xxsum0 - (((unsigned int)xsum0 * xsum0) >> 8));
}
unsigned int vp9_sub_pixel_mse16x16_wmt(
......@@ -417,7 +417,7 @@ unsigned int vp9_sub_pixel_variance16x8_wmt
}
*sse = xxsum0;
return (xxsum0 - ((xsum0 * xsum0) >> 7));
return (xxsum0 - (((unsigned int)xsum0 * xsum0) >> 7));
}
unsigned int vp9_sub_pixel_variance8x16_wmt
......@@ -457,7 +457,7 @@ unsigned int vp9_sub_pixel_variance8x16_wmt
}
*sse = xxsum;
return (xxsum - ((xsum * xsum) >> 7));
return (xxsum - (((unsigned int)xsum * xsum) >> 7));
}
......@@ -476,7 +476,7 @@ unsigned int vp9_variance_halfpixvar16x16_h_wmt(
&xsum0, &xxsum0);
*sse = xxsum0;
return (xxsum0 - ((xsum0 * xsum0) >> 8));
return (xxsum0 - (((unsigned int)xsum0 * xsum0) >> 8));
}
......@@ -494,7 +494,7 @@ unsigned int vp9_variance_halfpixvar16x16_v_wmt(
&xsum0, &xxsum0);
*sse = xxsum0;
return (xxsum0 - ((xsum0 * xsum0) >> 8));
return (xxsum0 - (((unsigned int)xsum0 * xsum0) >> 8));
}
......@@ -513,5 +513,5 @@ unsigned int vp9_variance_halfpixvar16x16_hv_wmt(
&xsum0, &xxsum0);
*sse = xxsum0;
return (xxsum0 - ((xsum0 * xsum0) >> 8));
return (xxsum0 - (((unsigned int)xsum0 * xsum0) >> 8));
}
......@@ -106,7 +106,7 @@ unsigned int vp9_sub_pixel_variance16x16_ssse3
}
*sse = xxsum0;
return (xxsum0 - ((xsum0 * xsum0) >> 8));
return (xxsum0 - (((unsigned int)xsum0 * xsum0) >> 8));
}
unsigned int vp9_sub_pixel_variance16x8_ssse3
......@@ -147,5 +147,5 @@ unsigned int vp9_sub_pixel_variance16x8_ssse3
}
*sse = xxsum0;
return (xxsum0 - ((xsum0 * xsum0) >> 7));
return (xxsum0 - (((unsigned int)xsum0 * xsum0) >> 7));
}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment