Merge "Fix variance (signed integer) overflow" into experimental

test/test.mk

@@ -6,5 +6,6 @@ LIBVPX_TEST_SRCS-yes += fdct4x4_test.cc
 LIBVPX_TEST_SRCS-yes += fdct8x8_test.cc
 LIBVPX_TEST_SRCS-yes += idct8x8_test.cc
 LIBVPX_TEST_SRCS-yes += test_libvpx.cc
+LIBVPX_TEST_SRCS-yes += variance_test.cc
 
 LIBVPX_TEST_DATA-yes += hantro_collage_w352h288.yuv

test/variance_test.cc

+/*
+ *  Copyright (c) 2012 The WebM project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+#include <stdlib.h>
+#include <new>
+
+#include "third_party/googletest/src/include/gtest/gtest.h"
+
+#include "vpx_config.h"
+extern "C" {
+#include "vp9/encoder/variance.h"
+#include "vpx/vpx_integer.h"
+#include "vpx_rtcd.h"
+}
+
+namespace {
+
+using ::std::tr1::get;
+using ::std::tr1::make_tuple;
+using ::std::tr1::tuple;
+
+class VP9VarianceTest :
+    public ::testing::TestWithParam<tuple<int, int, vp9_variance_fn_t> > {
+ public:
+  virtual void SetUp() {
+    const tuple<int, int, vp9_variance_fn_t>& params = GetParam();
+    width_  = get<0>(params);
+    height_ = get<1>(params);
+    variance_ = get<2>(params);
+
+    block_size_ = width_ * height_;
+    src_ = new uint8_t[block_size_];
+    ref_ = new uint8_t[block_size_];
+    ASSERT_TRUE(src_ != NULL);
+    ASSERT_TRUE(ref_ != NULL);
+  }
+
+  virtual void TearDown() {
+    delete[] src_;
+    delete[] ref_;
+  }
+
+ protected:
+  uint8_t* src_;
+  uint8_t* ref_;
+  int width_;
+  int height_;
+  int block_size_;
+  vp9_variance_fn_t variance_;
+};
+
+TEST_P(VP9VarianceTest, Zero) {
+  for (int i = 0; i <= 255; ++i) {
+    memset(src_, i, block_size_);
+    for (int j = 0; j <= 255; ++j) {
+      memset(ref_, j, block_size_);
+      unsigned int sse;
+      const unsigned int var = variance_(src_, width_, ref_, width_, &sse);
+      EXPECT_EQ(0u, var) << "src values: " << i << "ref values: " << j;
+    }
+  }
+}
+
+TEST_P(VP9VarianceTest, OneQuarter) {
+  memset(src_, 255, block_size_);
+  const int half = block_size_ / 2;
+  memset(ref_, 255, half);
+  memset(ref_ + half, 0, half);
+  unsigned int sse;
+  const unsigned int var = variance_(src_, width_, ref_, width_, &sse);
+  const unsigned int expected = block_size_ * 255 * 255 / 4;
+  EXPECT_EQ(expected, var);
+}
+
+const vp9_variance_fn_t variance4x4_c = vp9_variance4x4_c;
+const vp9_variance_fn_t variance8x8_c = vp9_variance8x8_c;
+const vp9_variance_fn_t variance8x16_c = vp9_variance8x16_c;
+const vp9_variance_fn_t variance16x8_c = vp9_variance16x8_c;
+const vp9_variance_fn_t variance16x16_c = vp9_variance16x16_c;
+INSTANTIATE_TEST_CASE_P(
+    C, VP9VarianceTest,
+    ::testing::Values(make_tuple(4, 4, variance4x4_c),
+                      make_tuple(8, 8, variance8x8_c),
+                      make_tuple(8, 16, variance8x16_c),
+                      make_tuple(16, 8, variance16x8_c),
+                      make_tuple(16, 16, variance16x16_c)));
+
+#if HAVE_MMX
+const vp9_variance_fn_t variance4x4_mmx = vp9_variance4x4_mmx;
+const vp9_variance_fn_t variance8x8_mmx = vp9_variance8x8_mmx;
+const vp9_variance_fn_t variance8x16_mmx = vp9_variance8x16_mmx;
+const vp9_variance_fn_t variance16x8_mmx = vp9_variance16x8_mmx;
+const vp9_variance_fn_t variance16x16_mmx = vp9_variance16x16_mmx;
+INSTANTIATE_TEST_CASE_P(
+    MMX, VP9VarianceTest,
+    ::testing::Values(make_tuple(4, 4, variance4x4_mmx),
+                      make_tuple(8, 8, variance8x8_mmx),
+                      make_tuple(8, 16, variance8x16_mmx),
+                      make_tuple(16, 8, variance16x8_mmx),
+                      make_tuple(16, 16, variance16x16_mmx)));
+#endif
+
+#if HAVE_SSE2
+const vp9_variance_fn_t variance4x4_wmt = vp9_variance4x4_wmt;
+const vp9_variance_fn_t variance8x8_wmt = vp9_variance8x8_wmt;
+const vp9_variance_fn_t variance8x16_wmt = vp9_variance8x16_wmt;
+const vp9_variance_fn_t variance16x8_wmt = vp9_variance16x8_wmt;
+const vp9_variance_fn_t variance16x16_wmt = vp9_variance16x16_wmt;
+INSTANTIATE_TEST_CASE_P(
+    SSE2, VP9VarianceTest,
+    ::testing::Values(make_tuple(4, 4, variance4x4_wmt),
+                      make_tuple(8, 8, variance8x8_wmt),
+                      make_tuple(8, 16, variance8x16_wmt),
+                      make_tuple(16, 8, variance16x8_wmt),
+                      make_tuple(16, 16, variance16x16_wmt)));
+#endif
+}  // namespace

vp9/common/findnearmv.c

@@ -197,7 +197,7 @@ unsigned int vp9_variance2x16_c(const unsigned char *src_ptr,
 
   variance(src_ptr, source_stride, ref_ptr, recon_stride, 2, 16, &var, &avg);
   *sse = var;
-  return (var - ((avg * avg) >> 5));
+  return (var - (((unsigned int)avg * avg) >> 5));
 }
 
 unsigned int vp9_variance16x2_c(const unsigned char *src_ptr,
@@ -210,7 +210,7 @@ unsigned int vp9_variance16x2_c(const unsigned char *src_ptr,
 
   variance(src_ptr, source_stride, ref_ptr, recon_stride, 16, 2, &var, &avg);
   *sse = var;
-  return (var - ((avg * avg) >> 5));
+  return (var - (((unsigned int)avg * avg) >> 5));
 }
 
 unsigned int vp9_sub_pixel_variance16x2_c(const unsigned char  *src_ptr,

vp9/encoder/variance_c.c

@@ -37,7 +37,8 @@ unsigned int vp9_variance32x32_c(const unsigned char *src_ptr,
 
   variance(src_ptr, source_stride, ref_ptr, recon_stride, 32, 32, &var, &avg);
   *sse = var;
-  return (var - ((avg * avg) >> 10));
+  // TODO(rbultje): in extreme cases these products will rollover.
+  return (var - (((unsigned int)avg * avg) >> 10));
 }
 #endif
 
@@ -51,7 +52,7 @@ unsigned int vp9_variance16x16_c(const unsigned char *src_ptr,
 
   variance(src_ptr, source_stride, ref_ptr, recon_stride, 16, 16, &var, &avg);
   *sse = var;
-  return (var - ((avg * avg) >> 8));
+  return (var - (((unsigned int)avg * avg) >> 8));
 }
 
 unsigned int vp9_variance8x16_c(const unsigned char *src_ptr,
@@ -64,7 +65,7 @@ unsigned int vp9_variance8x16_c(const unsigned char *src_ptr,
 
   variance(src_ptr, source_stride, ref_ptr, recon_stride, 8, 16, &var, &avg);
   *sse = var;
-  return (var - ((avg * avg) >> 7));
+  return (var - (((unsigned int)avg * avg) >> 7));
 }
 
 unsigned int vp9_variance16x8_c(const unsigned char *src_ptr,
@@ -77,7 +78,7 @@ unsigned int vp9_variance16x8_c(const unsigned char *src_ptr,
 
   variance(src_ptr, source_stride, ref_ptr, recon_stride, 16, 8, &var, &avg);
   *sse = var;
-  return (var - ((avg * avg) >> 7));
+  return (var - (((unsigned int)avg * avg) >> 7));
 }
 
 
@@ -91,7 +92,7 @@ unsigned int vp9_variance8x8_c(const unsigned char *src_ptr,
 
   variance(src_ptr, source_stride, ref_ptr, recon_stride, 8, 8, &var, &avg);
   *sse = var;
-  return (var - ((avg * avg) >> 6));
+  return (var - (((unsigned int)avg * avg) >> 6));
 }
 
 unsigned int vp9_variance4x4_c(const unsigned char *src_ptr,
@@ -104,7 +105,7 @@ unsigned int vp9_variance4x4_c(const unsigned char *src_ptr,
 
   variance(src_ptr, source_stride, ref_ptr, recon_stride, 4, 4, &var, &avg);
   *sse = var;
-  return (var - ((avg * avg) >> 4));
+  return (var - (((unsigned int)avg * avg) >> 4));
 }
 
 

vp9/encoder/x86/variance_mmx.c

@@ -89,7 +89,7 @@ unsigned int vp9_variance4x4_mmx(
 
   vp9_get4x4var_mmx(src_ptr, source_stride, ref_ptr, recon_stride, &var, &avg);
   *sse = var;
-  return (var - ((avg * avg) >> 4));
+  return (var - (((unsigned int)avg * avg) >> 4));
 
 }
 
@@ -105,7 +105,7 @@ unsigned int vp9_variance8x8_mmx(
   vp9_get8x8var_mmx(src_ptr, source_stride, ref_ptr, recon_stride, &var, &avg);
   *sse = var;
 
-  return (var - ((avg * avg) >> 6));
+  return (var - (((unsigned int)avg * avg) >> 6));
 
 }
 
@@ -148,7 +148,7 @@ unsigned int vp9_variance16x16_mmx(
   var = sse0 + sse1 + sse2 + sse3;
   avg = sum0 + sum1 + sum2 + sum3;
   *sse = var;
-  return (var - ((avg * avg) >> 8));
+  return (var - (((unsigned int)avg * avg) >> 8));
 }
 
 unsigned int vp9_variance16x8_mmx(
@@ -166,7 +166,7 @@ unsigned int vp9_variance16x8_mmx(
   var = sse0 + sse1;
   avg = sum0 + sum1;
   *sse = var;
-  return (var - ((avg * avg) >> 7));
+  return (var - (((unsigned int)avg * avg) >> 7));
 
 }
 
@@ -187,7 +187,7 @@ unsigned int vp9_variance8x16_mmx(
   avg = sum0 + sum1;
   *sse = var;
 
-  return (var - ((avg * avg) >> 7));
+  return (var - (((unsigned int)avg * avg) >> 7));
 
 }
 
@@ -237,7 +237,7 @@ unsigned int vp9_sub_pixel_variance4x4_mmx
     &xsum, &xxsum
   );
   *sse = xxsum;
-  return (xxsum - ((xsum * xsum) >> 4));
+  return (xxsum - (((unsigned int)xsum * xsum) >> 4));
 }
 
 
@@ -261,7 +261,7 @@ unsigned int vp9_sub_pixel_variance8x8_mmx
     &xsum, &xxsum
   );
   *sse = xxsum;
-  return (xxsum - ((xsum * xsum) >> 6));
+  return (xxsum - (((unsigned int)xsum * xsum) >> 6));
 }
 
 unsigned int vp9_sub_pixel_variance16x16_mmx
@@ -296,7 +296,7 @@ unsigned int vp9_sub_pixel_variance16x16_mmx
   xxsum0 += xxsum1;
 
   *sse = xxsum0;
-  return (xxsum0 - ((xsum0 * xsum0) >> 8));
+  return (xxsum0 - (((unsigned int)xsum0 * xsum0) >> 8));
 
 
 }
@@ -347,7 +347,7 @@ unsigned int vp9_sub_pixel_variance16x8_mmx
   xxsum0 += xxsum1;
 
   *sse = xxsum0;
-  return (xxsum0 - ((xsum0 * xsum0) >> 7));
+  return (xxsum0 - (((unsigned int)xsum0 * xsum0) >> 7));
 }
 
 unsigned int vp9_sub_pixel_variance8x16_mmx
@@ -369,7 +369,7 @@ unsigned int vp9_sub_pixel_variance8x16_mmx
     &xsum, &xxsum
   );
   *sse = xxsum;
-  return (xxsum - ((xsum * xsum) >> 7));
+  return (xxsum - (((unsigned int)xsum * xsum) >> 7));
 }
 
 

vp9/encoder/x86/variance_sse2.c

@@ -150,7 +150,7 @@ unsigned int vp9_variance4x4_wmt(
 
   vp9_get4x4var_mmx(src_ptr, source_stride, ref_ptr, recon_stride, &var, &avg);
   *sse = var;
-  return (var - ((avg * avg) >> 4));
+  return (var - (((unsigned int)avg * avg) >> 4));
 
 }
 
@@ -166,7 +166,7 @@ unsigned int vp9_variance8x8_wmt
 
   vp9_get8x8var_sse2(src_ptr, source_stride, ref_ptr, recon_stride, &var, &avg);
   *sse = var;
-  return (var - ((avg * avg) >> 6));
+  return (var - (((unsigned int)avg * avg) >> 6));
 
 }
 
@@ -184,7 +184,7 @@ unsigned int vp9_variance16x16_wmt
 
   vp9_get16x16var_sse2(src_ptr, source_stride, ref_ptr, recon_stride, &sse0, &sum0);
   *sse = sse0;
-  return (sse0 - ((sum0 * sum0) >> 8));
+  return (sse0 - (((unsigned int)sum0 * sum0) >> 8));
 }
 unsigned int vp9_mse16x16_wmt(
   const unsigned char *src_ptr,
@@ -218,7 +218,7 @@ unsigned int vp9_variance16x8_wmt
   var = sse0 + sse1;
   avg = sum0 + sum1;
   *sse = var;
-  return (var - ((avg * avg) >> 7));
+  return (var - (((unsigned int)avg * avg) >> 7));
 
 }
 
@@ -238,7 +238,7 @@ unsigned int vp9_variance8x16_wmt
   var = sse0 + sse1;
   avg = sum0 + sum1;
   *sse = var;
-  return (var - ((avg * avg) >> 7));
+  return (var - (((unsigned int)avg * avg) >> 7));
 
 }
 
@@ -261,7 +261,7 @@ unsigned int vp9_sub_pixel_variance4x4_wmt
     &xsum, &xxsum
   );
   *sse = xxsum;
-  return (xxsum - ((xsum * xsum) >> 4));
+  return (xxsum - (((unsigned int)xsum * xsum) >> 4));
 }
 
 
@@ -302,7 +302,7 @@ unsigned int vp9_sub_pixel_variance8x8_wmt
   }
 
   *sse = xxsum;
-  return (xxsum - ((xsum * xsum) >> 6));
+  return (xxsum - (((unsigned int)xsum * xsum) >> 6));
 }
 
 unsigned int vp9_sub_pixel_variance16x16_wmt
@@ -355,7 +355,7 @@ unsigned int vp9_sub_pixel_variance16x16_wmt
   }
 
   *sse = xxsum0;
-  return (xxsum0 - ((xsum0 * xsum0) >> 8));
+  return (xxsum0 - (((unsigned int)xsum0 * xsum0) >> 8));
 }
 
 unsigned int vp9_sub_pixel_mse16x16_wmt(
@@ -417,7 +417,7 @@ unsigned int vp9_sub_pixel_variance16x8_wmt
   }
 
   *sse = xxsum0;
-  return (xxsum0 - ((xsum0 * xsum0) >> 7));
+  return (xxsum0 - (((unsigned int)xsum0 * xsum0) >> 7));
 }
 
 unsigned int vp9_sub_pixel_variance8x16_wmt
@@ -457,7 +457,7 @@ unsigned int vp9_sub_pixel_variance8x16_wmt
   }
 
   *sse = xxsum;
-  return (xxsum - ((xsum * xsum) >> 7));
+  return (xxsum - (((unsigned int)xsum * xsum) >> 7));
 }
 
 
@@ -476,7 +476,7 @@ unsigned int vp9_variance_halfpixvar16x16_h_wmt(
     &xsum0, &xxsum0);
 
   *sse = xxsum0;
-  return (xxsum0 - ((xsum0 * xsum0) >> 8));
+  return (xxsum0 - (((unsigned int)xsum0 * xsum0) >> 8));
 }
 
 
@@ -494,7 +494,7 @@ unsigned int vp9_variance_halfpixvar16x16_v_wmt(
     &xsum0, &xxsum0);
 
   *sse = xxsum0;
-  return (xxsum0 - ((xsum0 * xsum0) >> 8));
+  return (xxsum0 - (((unsigned int)xsum0 * xsum0) >> 8));
 }
 
 
@@ -513,5 +513,5 @@ unsigned int vp9_variance_halfpixvar16x16_hv_wmt(
     &xsum0, &xxsum0);
 
   *sse = xxsum0;
-  return (xxsum0 - ((xsum0 * xsum0) >> 8));
+  return (xxsum0 - (((unsigned int)xsum0 * xsum0) >> 8));
 }

vp9/encoder/x86/variance_ssse3.c

@@ -106,7 +106,7 @@ unsigned int vp9_sub_pixel_variance16x16_ssse3
   }
 
   *sse = xxsum0;
-  return (xxsum0 - ((xsum0 * xsum0) >> 8));
+  return (xxsum0 - (((unsigned int)xsum0 * xsum0) >> 8));
 }
 
 unsigned int vp9_sub_pixel_variance16x8_ssse3
@@ -147,5 +147,5 @@ unsigned int vp9_sub_pixel_variance16x8_ssse3
   }
 
   *sse = xxsum0;
-  return (xxsum0 - ((xsum0 * xsum0) >> 7));
+  return (xxsum0 - (((unsigned int)xsum0 * xsum0) >> 7));
 }