Add SSE4.1 vpx_obmc_sad* implementations.

Speedup for these functions: 4x

Change-Id: I21baa04f53c6ab308ea3edf3ebacc62970e97454

test/obmc_sad_test.cc

+/*
+ *  Copyright (c) 2016 The WebM project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "third_party/googletest/src/include/gtest/gtest.h"
+#include "test/acm_random.h"
+
+#include "test/function_equivalence_test.h"
+
+#include "./vpx_config.h"
+#include "./vpx_dsp_rtcd.h"
+#include "vpx/vpx_integer.h"
+
+#define MAX_SB_SQUARE (MAX_SB_SIZE * MAX_SB_SIZE)
+
+using std::tr1::make_tuple;
+
+using libvpx_test::ACMRandom;
+using libvpx_test::FunctionEquivalenceTest;
+
+namespace {
+
+static const int kIterations = 1000;
+static const int kMaskMax = 64;
+
+typedef unsigned int (*ObmcSadF)(const uint8_t *ref, int ref_stride,
+                                 const int32_t *wsrc, const int32_t *mask);
+
+////////////////////////////////////////////////////////////////////////////////
+// 8 bit
+////////////////////////////////////////////////////////////////////////////////
+
+class ObmcSadTest : public FunctionEquivalenceTest<ObmcSadF> {
+ public:
+  ObmcSadTest() : rng_(ACMRandom::DeterministicSeed()) {}
+
+ protected:
+  ACMRandom rng_;
+};
+
+TEST_P(ObmcSadTest, RandomValues) {
+  DECLARE_ALIGNED(32, uint8_t, ref[MAX_SB_SQUARE]);
+  DECLARE_ALIGNED(32, int32_t, wsrc[MAX_SB_SQUARE]);
+  DECLARE_ALIGNED(32, int32_t, mask[MAX_SB_SQUARE]);
+
+  for (int iter = 0 ; iter < kIterations && !HasFatalFailure() ; ++iter) {
+    const int ref_stride = rng_(MAX_SB_SIZE + 1);
+
+    for (int i = 0 ; i < MAX_SB_SQUARE ; ++i) {
+      ref[i] = rng_.Rand8();
+      wsrc[i] = rng_.Rand8() * rng_(kMaskMax * kMaskMax + 1);
+      mask[i] = rng_(kMaskMax * kMaskMax + 1);
+    }
+
+    const unsigned int ref_res = ref_func_(ref, ref_stride, wsrc, mask);
+    const unsigned int tst_res = tst_func_(ref, ref_stride, wsrc, mask);
+
+    ASSERT_EQ(ref_res, tst_res);
+  }
+}
+
+TEST_P(ObmcSadTest, ExtremeValues) {
+  DECLARE_ALIGNED(32, uint8_t, ref[MAX_SB_SQUARE]);
+  DECLARE_ALIGNED(32, int32_t, wsrc[MAX_SB_SQUARE]);
+  DECLARE_ALIGNED(32, int32_t, mask[MAX_SB_SQUARE]);
+
+  for (int iter = 0 ; iter < MAX_SB_SIZE && !HasFatalFailure() ; ++iter) {
+    const int ref_stride = iter;
+
+    for (int i = 0 ; i < MAX_SB_SQUARE ; ++i) {
+      ref[i] = UINT8_MAX;
+      wsrc[i] = UINT8_MAX * kMaskMax * kMaskMax;
+      mask[i] = kMaskMax * kMaskMax;
+    }
+
+    const unsigned int ref_res = ref_func_(ref, ref_stride, wsrc, mask);
+    const unsigned int tst_res = tst_func_(ref, ref_stride, wsrc, mask);
+
+    ASSERT_EQ(ref_res, tst_res);
+  }
+}
+
+#if HAVE_SSE4_1
+const ObmcSadTest::ParamType sse4_functions[] = {
+#if CONFIG_EXT_PARTITION
+  make_tuple(vpx_obmc_sad128x128_c, vpx_obmc_sad128x128_sse4_1),
+  make_tuple(vpx_obmc_sad128x64_c, vpx_obmc_sad128x64_sse4_1),
+  make_tuple(vpx_obmc_sad64x128_c, vpx_obmc_sad64x128_sse4_1),
+#endif  // CONFIG_EXT_PARTITION
+  make_tuple(vpx_obmc_sad64x64_c, vpx_obmc_sad64x64_sse4_1),
+  make_tuple(vpx_obmc_sad64x32_c, vpx_obmc_sad64x32_sse4_1),
+  make_tuple(vpx_obmc_sad32x64_c, vpx_obmc_sad32x64_sse4_1),
+  make_tuple(vpx_obmc_sad32x32_c, vpx_obmc_sad32x32_sse4_1),
+  make_tuple(vpx_obmc_sad32x16_c, vpx_obmc_sad32x16_sse4_1),
+  make_tuple(vpx_obmc_sad16x32_c, vpx_obmc_sad16x32_sse4_1),
+  make_tuple(vpx_obmc_sad16x16_c, vpx_obmc_sad16x16_sse4_1),
+  make_tuple(vpx_obmc_sad16x8_c, vpx_obmc_sad16x8_sse4_1),
+  make_tuple(vpx_obmc_sad8x16_c, vpx_obmc_sad8x16_sse4_1),
+  make_tuple(vpx_obmc_sad8x8_c, vpx_obmc_sad8x8_sse4_1),
+  make_tuple(vpx_obmc_sad8x4_c, vpx_obmc_sad8x4_sse4_1),
+  make_tuple(vpx_obmc_sad4x8_c, vpx_obmc_sad4x8_sse4_1),
+  make_tuple(vpx_obmc_sad4x4_c, vpx_obmc_sad4x4_sse4_1)
+};
+
+INSTANTIATE_TEST_CASE_P(SSE4_1_C_COMPARE, ObmcSadTest,
+                        ::testing::ValuesIn(sse4_functions));
+#endif  // HAVE_SSE4_1
+
+////////////////////////////////////////////////////////////////////////////////
+// High bit-depth
+////////////////////////////////////////////////////////////////////////////////
+
+#if CONFIG_VP9_HIGHBITDEPTH
+class ObmcSadHBDTest : public FunctionEquivalenceTest<ObmcSadF> {
+ public:
+  ObmcSadHBDTest() : rng_(ACMRandom::DeterministicSeed()) {}
+
+ protected:
+  ACMRandom rng_;
+};
+
+TEST_P(ObmcSadHBDTest, RandomValues) {
+  DECLARE_ALIGNED(32, uint16_t, ref[MAX_SB_SQUARE]);
+  DECLARE_ALIGNED(32, int32_t, wsrc[MAX_SB_SQUARE]);
+  DECLARE_ALIGNED(32, int32_t, mask[MAX_SB_SQUARE]);
+
+  for (int iter = 0 ; iter < kIterations && !HasFatalFailure() ; ++iter) {
+    const int ref_stride = rng_(MAX_SB_SIZE + 1);
+
+    for (int i = 0 ; i < MAX_SB_SQUARE ; ++i) {
+      ref[i] = rng_(1<<12);
+      wsrc[i] = rng_(1<<12) * rng_(kMaskMax * kMaskMax + 1);
+      mask[i] = rng_(kMaskMax * kMaskMax + 1);
+    }
+
+    const unsigned int ref_res = ref_func_(CONVERT_TO_BYTEPTR(ref), ref_stride,
+                                           wsrc, mask);
+    const unsigned int tst_res = tst_func_(CONVERT_TO_BYTEPTR(ref), ref_stride,
+                                           wsrc, mask);
+
+    ASSERT_EQ(ref_res, tst_res);
+  }
+}
+
+TEST_P(ObmcSadHBDTest, ExtremeValues) {
+  DECLARE_ALIGNED(32, uint16_t, ref[MAX_SB_SQUARE]);
+  DECLARE_ALIGNED(32, int32_t, wsrc[MAX_SB_SQUARE]);
+  DECLARE_ALIGNED(32, int32_t, mask[MAX_SB_SQUARE]);
+
+  for (int iter = 0 ; iter < MAX_SB_SIZE && !HasFatalFailure() ; ++iter) {
+    const int ref_stride = iter;
+
+    for (int i = 0 ; i < MAX_SB_SQUARE ; ++i) {
+      ref[i] = (1 << 12) - 1;
+      wsrc[i] = ((1 << 12) - 1) * kMaskMax * kMaskMax;
+      mask[i] = kMaskMax * kMaskMax;
+    }
+
+    const unsigned int ref_res = ref_func_(CONVERT_TO_BYTEPTR(ref), ref_stride,
+                                           wsrc, mask);
+    const unsigned int tst_res = tst_func_(CONVERT_TO_BYTEPTR(ref), ref_stride,
+                                           wsrc, mask);
+
+    ASSERT_EQ(ref_res, tst_res);
+  }
+}
+
+#if HAVE_SSE4_1
+ObmcSadHBDTest::ParamType sse4_functions_hbd[] = {
+#if CONFIG_EXT_PARTITION
+  make_tuple(vpx_highbd_obmc_sad128x128_c, vpx_highbd_obmc_sad128x128_sse4_1),
+  make_tuple(vpx_highbd_obmc_sad128x64_c, vpx_highbd_obmc_sad128x64_sse4_1),
+  make_tuple(vpx_highbd_obmc_sad64x128_c, vpx_highbd_obmc_sad64x128_sse4_1),
+#endif  // CONFIG_EXT_PARTITION
+  make_tuple(vpx_highbd_obmc_sad64x64_c, vpx_highbd_obmc_sad64x64_sse4_1),
+  make_tuple(vpx_highbd_obmc_sad64x32_c, vpx_highbd_obmc_sad64x32_sse4_1),
+  make_tuple(vpx_highbd_obmc_sad32x64_c, vpx_highbd_obmc_sad32x64_sse4_1),
+  make_tuple(vpx_highbd_obmc_sad32x32_c, vpx_highbd_obmc_sad32x32_sse4_1),
+  make_tuple(vpx_highbd_obmc_sad32x16_c, vpx_highbd_obmc_sad32x16_sse4_1),
+  make_tuple(vpx_highbd_obmc_sad16x32_c, vpx_highbd_obmc_sad16x32_sse4_1),
+  make_tuple(vpx_highbd_obmc_sad16x16_c, vpx_highbd_obmc_sad16x16_sse4_1),
+  make_tuple(vpx_highbd_obmc_sad16x8_c, vpx_highbd_obmc_sad16x8_sse4_1),
+  make_tuple(vpx_highbd_obmc_sad8x16_c, vpx_highbd_obmc_sad8x16_sse4_1),
+  make_tuple(vpx_highbd_obmc_sad8x8_c, vpx_highbd_obmc_sad8x8_sse4_1),
+  make_tuple(vpx_highbd_obmc_sad8x4_c, vpx_highbd_obmc_sad8x4_sse4_1),
+  make_tuple(vpx_highbd_obmc_sad4x8_c, vpx_highbd_obmc_sad4x8_sse4_1),
+  make_tuple(vpx_highbd_obmc_sad4x4_c, vpx_highbd_obmc_sad4x4_sse4_1)
+};
+
+INSTANTIATE_TEST_CASE_P(SSE4_1_C_COMPARE, ObmcSadHBDTest,
+                        ::testing::ValuesIn(sse4_functions_hbd));
+#endif  // HAVE_SSE4_1
+#endif  // CONFIG_VP9_HIGHBITDEPTH
+}  // namespace

test/test.mk

@@ -187,6 +187,10 @@ LIBVPX_TEST_SRCS-$(HAVE_SSSE3) += masked_sad_test.cc
 LIBVPX_TEST_SRCS-$(CONFIG_VP10_ENCODER) += blend_mask6_test.cc
 endif
 
+ifeq ($(CONFIG_OBMC),yes)
+LIBVPX_TEST_SRCS-$(CONFIG_VP10_ENCODER) += obmc_sad_test.cc
+endif
+
 ifeq ($(CONFIG_VP9_HIGHBITDEPTH),yes)
 LIBVPX_TEST_SRCS-$(HAVE_SSE4_1) += vp10_highbd_iht_test.cc
 endif # CONFIG_VP9_HIGHBITDEPTH

vpx_dsp/vpx_dsp.mk

@@ -314,6 +314,9 @@ ifeq ($(CONFIG_EXT_INTER),yes)
 DSP_SRCS-$(HAVE_SSSE3)  += x86/masked_sad_intrin_ssse3.c
 DSP_SRCS-$(HAVE_SSSE3)  += x86/masked_variance_intrin_ssse3.c
 endif  #CONFIG_EXT_INTER
+ifeq ($(CONFIG_OBMC),yes)
+DSP_SRCS-$(HAVE_SSE4_1) += x86/obmc_sad_sse4.c
+endif  #CONFIG_OBMC
 endif  #CONFIG_VP10_ENCODER
 
 ifeq ($(CONFIG_USE_X86INC),yes)

vpx_dsp/vpx_dsp_rtcd_defs.pl

@@ -1104,14 +1104,14 @@ if (vpx_config("CONFIG_OBMC") eq "yes") {
   foreach (@block_sizes) {
     ($w, $h) = @$_;
     add_proto qw/unsigned int/, "vpx_obmc_sad${w}x${h}", "const uint8_t *ref_ptr, int ref_stride, const int32_t *wsrc_ptr, const int32_t *mask";
-    specialize "vpx_obmc_sad${w}x${h}";
+    specialize "vpx_obmc_sad${w}x${h}", qw/sse4_1/;
   }
 
   if (vpx_config("CONFIG_VP9_HIGHBITDEPTH") eq "yes") {
     foreach (@block_sizes) {
       ($w, $h) = @$_;
       add_proto qw/unsigned int/, "vpx_highbd_obmc_sad${w}x${h}", "const uint8_t *ref_ptr, int ref_stride, const int32_t *wsrc_ptr, const int32_t *mask";
-      specialize "vpx_highbd_obmc_sad${w}x${h}";
+      specialize "vpx_highbd_obmc_sad${w}x${h}", qw/sse4_1/;
     }
   }
 }

vpx_dsp/x86/obmc_sad_sse4.c

+/*
+ *  Copyright (c) 2016 The WebM project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include <assert.h>
+#include <immintrin.h>
+
+#include "./vpx_config.h"
+#include "vpx_ports/mem.h"
+#include "vpx/vpx_integer.h"
+
+#include "vpx_dsp/x86/synonyms.h"
+
+////////////////////////////////////////////////////////////////////////////////
+// 8 bit
+////////////////////////////////////////////////////////////////////////////////
+
+static INLINE unsigned int obmc_sad_w4(const uint8_t *a, const int a_stride,
+                                       const int32_t *b, const int32_t *m,
+                                       const int height) {
+  const int a_step = a_stride - 4;
+  int n = 0;
+  __m128i v_sad_d = _mm_setzero_si128();
+
+  do {
+    const __m128i v_a_b = xx_loadl_32(a + n);
+    const __m128i v_m_d = xx_load_128(m + n);
+    const __m128i v_b_d = xx_load_128(b + n);
+
+    const __m128i v_a_d = _mm_cvtepu8_epi32(v_a_b);
+
+    // Values in both a and m fit in 15 bits, and are packed at 32 bit
+    // boundaries. We use pmaddwd, as it has lower latency on Haswell
+    // than pmulld but produces the same result with these inputs.
+    const __m128i v_am_d = _mm_madd_epi16(v_a_d, v_m_d);
+
+    const __m128i v_diff_d = _mm_sub_epi32(v_b_d, v_am_d);
+    const __m128i v_absdiff_d = _mm_abs_epi32(v_diff_d);
+
+    // Rounded absolute difference
+    const __m128i v_rad_d = xx_roundn_epu32(v_absdiff_d, 12);
+
+    v_sad_d = _mm_add_epi32(v_sad_d, v_rad_d);
+
+    n += 4;
+
+    if (n % 4 == 0)
+      a += a_step;
+  } while (n < 4 * height);
+
+  return xx_hsum_epi32_si32(v_sad_d);
+}
+
+static INLINE unsigned int obmc_sad_w8n(const uint8_t *a, const int a_stride,
+                                        const int32_t *b, const int32_t *m,
+                                        const int width, const int height) {
+  const int a_step = a_stride - width;
+  int n = 0;
+  __m128i v_sad_d = _mm_setzero_si128();
+  assert(width >= 8 && (width & (width - 1)) == 0);
+
+  do {
+    const __m128i v_a1_b = xx_loadl_32(a + n + 4);
+    const __m128i v_m1_d = xx_load_128(m + n + 4);
+    const __m128i v_b1_d = xx_load_128(b + n + 4);
+    const __m128i v_a0_b = xx_loadl_32(a + n);
+    const __m128i v_m0_d = xx_load_128(m + n);
+    const __m128i v_b0_d = xx_load_128(b + n);
+
+    const __m128i v_a0_d = _mm_cvtepu8_epi32(v_a0_b);
+    const __m128i v_a1_d = _mm_cvtepu8_epi32(v_a1_b);
+
+    // Values in both a and m fit in 15 bits, and are packed at 32 bit
+    // boundaries. We use pmaddwd, as it has lower latency on Haswell
+    // than pmulld but produces the same result with these inputs.
+    const __m128i v_am0_d = _mm_madd_epi16(v_a0_d, v_m0_d);
+    const __m128i v_am1_d = _mm_madd_epi16(v_a1_d, v_m1_d);
+
+    const __m128i v_diff0_d = _mm_sub_epi32(v_b0_d, v_am0_d);
+    const __m128i v_diff1_d = _mm_sub_epi32(v_b1_d, v_am1_d);
+    const __m128i v_absdiff0_d = _mm_abs_epi32(v_diff0_d);
+    const __m128i v_absdiff1_d = _mm_abs_epi32(v_diff1_d);
+
+    // Rounded absolute difference
+    const __m128i v_rad0_d = xx_roundn_epu32(v_absdiff0_d, 12);
+    const __m128i v_rad1_d = xx_roundn_epu32(v_absdiff1_d, 12);
+
+    v_sad_d = _mm_add_epi32(v_sad_d, v_rad0_d);
+    v_sad_d = _mm_add_epi32(v_sad_d, v_rad1_d);
+
+    n += 8;
+
+    if (n % width == 0)
+      a += a_step;
+  } while (n < width * height);
+
+  return xx_hsum_epi32_si32(v_sad_d);
+}
+
+#define OBMCSADWXH(w, h)                                                      \
+unsigned int vpx_obmc_sad##w##x##h##_sse4_1(const uint8_t *ref,               \
+                                            int ref_stride,                   \
+                                            const int32_t *wsrc,              \
+                                            const int32_t *msk) {             \
+  if (w == 4)                                                                 \
+    return obmc_sad_w4(ref, ref_stride, wsrc, msk, h);                        \
+  else                                                                        \
+    return obmc_sad_w8n(ref, ref_stride, wsrc, msk, w, h);                    \
+}
+
+#if CONFIG_EXT_PARTITION
+OBMCSADWXH(128, 128)
+OBMCSADWXH(128, 64)
+OBMCSADWXH(64, 128)
+#endif  // CONFIG_EXT_PARTITION
+OBMCSADWXH(64, 64)
+OBMCSADWXH(64, 32)
+OBMCSADWXH(32, 64)
+OBMCSADWXH(32, 32)
+OBMCSADWXH(32, 16)
+OBMCSADWXH(16, 32)
+OBMCSADWXH(16, 16)
+OBMCSADWXH(16, 8)
+OBMCSADWXH(8, 16)
+OBMCSADWXH(8, 8)
+OBMCSADWXH(8, 4)
+OBMCSADWXH(4, 8)
+OBMCSADWXH(4, 4)
+
+////////////////////////////////////////////////////////////////////////////////
+// High bit-depth
+////////////////////////////////////////////////////////////////////////////////
+
+#if CONFIG_VP9_HIGHBITDEPTH
+static INLINE unsigned int hbd_obmc_sad_w4(const uint8_t *a8,
+                                           const int a_stride,
+                                           const int32_t *b, const int32_t *m,
+                                           const int height) {
+  const uint16_t *a = CONVERT_TO_SHORTPTR(a8);
+  const int a_step = a_stride - 4;
+  int n = 0;
+  __m128i v_sad_d = _mm_setzero_si128();
+
+  do {
+    const __m128i v_a_w = xx_loadl_64(a + n);
+    const __m128i v_m_d = xx_load_128(m + n);
+    const __m128i v_b_d = xx_load_128(b + n);
+
+    const __m128i v_a_d = _mm_cvtepu16_epi32(v_a_w);
+
+    // Values in both a and m fit in 15 bits, and are packed at 32 bit
+    // boundaries. We use pmaddwd, as it has lower latency on Haswell
+    // than pmulld but produces the same result with these inputs.
+    const __m128i v_am_d = _mm_madd_epi16(v_a_d, v_m_d);
+
+    const __m128i v_diff_d = _mm_sub_epi32(v_b_d, v_am_d);
+    const __m128i v_absdiff_d = _mm_abs_epi32(v_diff_d);
+
+    // Rounded absolute difference
+    const __m128i v_rad_d = xx_roundn_epu32(v_absdiff_d, 12);
+
+    v_sad_d = _mm_add_epi32(v_sad_d, v_rad_d);
+
+    n += 4;
+
+    if (n % 4 == 0)
+      a += a_step;
+  } while (n < 4 * height);
+
+  return xx_hsum_epi32_si32(v_sad_d);
+}
+
+static INLINE unsigned int hbd_obmc_sad_w8n(const uint8_t *a8,
+                                            const int a_stride,
+                                            const int32_t *b, const int32_t *m,
+                                            const int width, const int height) {
+  const uint16_t *a = CONVERT_TO_SHORTPTR(a8);
+  const int a_step = a_stride - width;
+  int n = 0;
+  __m128i v_sad_d = _mm_setzero_si128();
+  assert(width >= 8 && (width & (width - 1)) == 0);
+
+  do {
+    const __m128i v_a1_w = xx_loadl_64(a + n + 4);
+    const __m128i v_m1_d = xx_load_128(m + n + 4);
+    const __m128i v_b1_d = xx_load_128(b + n + 4);
+    const __m128i v_a0_w = xx_loadl_64(a + n);
+    const __m128i v_m0_d = xx_load_128(m + n);
+    const __m128i v_b0_d = xx_load_128(b + n);
+
+    const __m128i v_a0_d = _mm_cvtepu16_epi32(v_a0_w);
+    const __m128i v_a1_d = _mm_cvtepu16_epi32(v_a1_w);
+
+    // Values in both a and m fit in 15 bits, and are packed at 32 bit
+    // boundaries. We use pmaddwd, as it has lower latency on Haswell
+    // than pmulld but produces the same result with these inputs.
+    const __m128i v_am0_d = _mm_madd_epi16(v_a0_d, v_m0_d);
+    const __m128i v_am1_d = _mm_madd_epi16(v_a1_d, v_m1_d);
+
+    const __m128i v_diff0_d = _mm_sub_epi32(v_b0_d, v_am0_d);
+    const __m128i v_diff1_d = _mm_sub_epi32(v_b1_d, v_am1_d);
+    const __m128i v_absdiff0_d = _mm_abs_epi32(v_diff0_d);
+    const __m128i v_absdiff1_d = _mm_abs_epi32(v_diff1_d);
+
+    // Rounded absolute difference
+    const __m128i v_rad0_d = xx_roundn_epu32(v_absdiff0_d, 12);
+    const __m128i v_rad1_d = xx_roundn_epu32(v_absdiff1_d, 12);
+
+    v_sad_d = _mm_add_epi32(v_sad_d, v_rad0_d);
+    v_sad_d = _mm_add_epi32(v_sad_d, v_rad1_d);
+
+    n += 8;
+
+    if (n % width == 0)
+      a += a_step;
+  } while (n < width * height);
+
+  return xx_hsum_epi32_si32(v_sad_d);
+}
+
+#define HBD_OBMCSADWXH(w, h)                                                  \
+unsigned int vpx_highbd_obmc_sad##w##x##h##_sse4_1(const uint8_t *ref,        \
+                                                   int ref_stride,            \
+                                                   const int32_t *wsrc,       \
+                                                   const int32_t *msk) {      \
+  if (w == 4)                                                                 \
+    return hbd_obmc_sad_w4(ref, ref_stride, wsrc, msk, h);                    \
+  else                                                                        \
+    return hbd_obmc_sad_w8n(ref, ref_stride, wsrc, msk, w, h);                \
+}
+
+#if CONFIG_EXT_PARTITION
+HBD_OBMCSADWXH(128, 128)
+HBD_OBMCSADWXH(128, 64)
+HBD_OBMCSADWXH(64, 128)
+#endif  // CONFIG_EXT_PARTITION
+HBD_OBMCSADWXH(64, 64)
+HBD_OBMCSADWXH(64, 32)
+HBD_OBMCSADWXH(32, 64)
+HBD_OBMCSADWXH(32, 32)
+HBD_OBMCSADWXH(32, 16)
+HBD_OBMCSADWXH(16, 32)
+HBD_OBMCSADWXH(16, 16)
+HBD_OBMCSADWXH(16, 8)
+HBD_OBMCSADWXH(8, 16)
+HBD_OBMCSADWXH(8, 8)
+HBD_OBMCSADWXH(8, 4)
+HBD_OBMCSADWXH(4, 8)
+HBD_OBMCSADWXH(4, 4)
+#endif  // CONFIG_VP9_HIGHBITDEPTH

vpx_dsp/x86/synonyms.h

@@ -66,4 +66,18 @@ static INLINE __m128i xx_roundn_epu16(__m128i v_val_w, int bits) {
   return _mm_avg_epu16(v_s_w, _mm_setzero_si128());
 }
 
+static INLINE __m128i xx_roundn_epu32(__m128i v_val_d, int bits) {
+  const __m128i v_bias_d = _mm_set1_epi32(1 << (bits - 1));
+  const __m128i v_tmp_d = _mm_add_epi32(v_val_d, v_bias_d);
+  return _mm_srli_epi32(v_tmp_d, bits);
+}
+
+#ifdef __SSSE3__
+static INLINE int32_t xx_hsum_epi32_si32(__m128i v_d) {
+  v_d = _mm_hadd_epi32(v_d, v_d);
+  v_d = _mm_hadd_epi32(v_d, v_d);
+  return _mm_cvtsi128_si32(v_d);
+}
+#endif  // __SSSE3__
+
 #endif  // VPX_DSP_X86_SYNONYS_H_