Commit aacdf98c authored by Geza Lore's avatar Geza Lore Committed by Pascal Massimino
Browse files

Add SSE4.1 vpx_obmc_sad* implementations.

Speedup for these functions: 4x

Change-Id: I21baa04f53c6ab308ea3edf3ebacc62970e97454
parent 007aa7dd
/*
* Copyright (c) 2016 The WebM project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "third_party/googletest/src/include/gtest/gtest.h"
#include "test/acm_random.h"
#include "test/function_equivalence_test.h"
#include "./vpx_config.h"
#include "./vpx_dsp_rtcd.h"
#include "vpx/vpx_integer.h"
#define MAX_SB_SQUARE (MAX_SB_SIZE * MAX_SB_SIZE)
using std::tr1::make_tuple;
using libvpx_test::ACMRandom;
using libvpx_test::FunctionEquivalenceTest;
namespace {
static const int kIterations = 1000;
static const int kMaskMax = 64;
typedef unsigned int (*ObmcSadF)(const uint8_t *ref, int ref_stride,
const int32_t *wsrc, const int32_t *mask);
////////////////////////////////////////////////////////////////////////////////
// 8 bit
////////////////////////////////////////////////////////////////////////////////
class ObmcSadTest : public FunctionEquivalenceTest<ObmcSadF> {
public:
ObmcSadTest() : rng_(ACMRandom::DeterministicSeed()) {}
protected:
ACMRandom rng_;
};
TEST_P(ObmcSadTest, RandomValues) {
DECLARE_ALIGNED(32, uint8_t, ref[MAX_SB_SQUARE]);
DECLARE_ALIGNED(32, int32_t, wsrc[MAX_SB_SQUARE]);
DECLARE_ALIGNED(32, int32_t, mask[MAX_SB_SQUARE]);
for (int iter = 0 ; iter < kIterations && !HasFatalFailure() ; ++iter) {
const int ref_stride = rng_(MAX_SB_SIZE + 1);
for (int i = 0 ; i < MAX_SB_SQUARE ; ++i) {
ref[i] = rng_.Rand8();
wsrc[i] = rng_.Rand8() * rng_(kMaskMax * kMaskMax + 1);
mask[i] = rng_(kMaskMax * kMaskMax + 1);
}
const unsigned int ref_res = ref_func_(ref, ref_stride, wsrc, mask);
const unsigned int tst_res = tst_func_(ref, ref_stride, wsrc, mask);
ASSERT_EQ(ref_res, tst_res);
}
}
TEST_P(ObmcSadTest, ExtremeValues) {
DECLARE_ALIGNED(32, uint8_t, ref[MAX_SB_SQUARE]);
DECLARE_ALIGNED(32, int32_t, wsrc[MAX_SB_SQUARE]);
DECLARE_ALIGNED(32, int32_t, mask[MAX_SB_SQUARE]);
for (int iter = 0 ; iter < MAX_SB_SIZE && !HasFatalFailure() ; ++iter) {
const int ref_stride = iter;
for (int i = 0 ; i < MAX_SB_SQUARE ; ++i) {
ref[i] = UINT8_MAX;
wsrc[i] = UINT8_MAX * kMaskMax * kMaskMax;
mask[i] = kMaskMax * kMaskMax;
}
const unsigned int ref_res = ref_func_(ref, ref_stride, wsrc, mask);
const unsigned int tst_res = tst_func_(ref, ref_stride, wsrc, mask);
ASSERT_EQ(ref_res, tst_res);
}
}
#if HAVE_SSE4_1
const ObmcSadTest::ParamType sse4_functions[] = {
#if CONFIG_EXT_PARTITION
make_tuple(vpx_obmc_sad128x128_c, vpx_obmc_sad128x128_sse4_1),
make_tuple(vpx_obmc_sad128x64_c, vpx_obmc_sad128x64_sse4_1),
make_tuple(vpx_obmc_sad64x128_c, vpx_obmc_sad64x128_sse4_1),
#endif // CONFIG_EXT_PARTITION
make_tuple(vpx_obmc_sad64x64_c, vpx_obmc_sad64x64_sse4_1),
make_tuple(vpx_obmc_sad64x32_c, vpx_obmc_sad64x32_sse4_1),
make_tuple(vpx_obmc_sad32x64_c, vpx_obmc_sad32x64_sse4_1),
make_tuple(vpx_obmc_sad32x32_c, vpx_obmc_sad32x32_sse4_1),
make_tuple(vpx_obmc_sad32x16_c, vpx_obmc_sad32x16_sse4_1),
make_tuple(vpx_obmc_sad16x32_c, vpx_obmc_sad16x32_sse4_1),
make_tuple(vpx_obmc_sad16x16_c, vpx_obmc_sad16x16_sse4_1),
make_tuple(vpx_obmc_sad16x8_c, vpx_obmc_sad16x8_sse4_1),
make_tuple(vpx_obmc_sad8x16_c, vpx_obmc_sad8x16_sse4_1),
make_tuple(vpx_obmc_sad8x8_c, vpx_obmc_sad8x8_sse4_1),
make_tuple(vpx_obmc_sad8x4_c, vpx_obmc_sad8x4_sse4_1),
make_tuple(vpx_obmc_sad4x8_c, vpx_obmc_sad4x8_sse4_1),
make_tuple(vpx_obmc_sad4x4_c, vpx_obmc_sad4x4_sse4_1)
};
INSTANTIATE_TEST_CASE_P(SSE4_1_C_COMPARE, ObmcSadTest,
::testing::ValuesIn(sse4_functions));
#endif // HAVE_SSE4_1
////////////////////////////////////////////////////////////////////////////////
// High bit-depth
////////////////////////////////////////////////////////////////////////////////
#if CONFIG_VP9_HIGHBITDEPTH
class ObmcSadHBDTest : public FunctionEquivalenceTest<ObmcSadF> {
public:
ObmcSadHBDTest() : rng_(ACMRandom::DeterministicSeed()) {}
protected:
ACMRandom rng_;
};
TEST_P(ObmcSadHBDTest, RandomValues) {
DECLARE_ALIGNED(32, uint16_t, ref[MAX_SB_SQUARE]);
DECLARE_ALIGNED(32, int32_t, wsrc[MAX_SB_SQUARE]);
DECLARE_ALIGNED(32, int32_t, mask[MAX_SB_SQUARE]);
for (int iter = 0 ; iter < kIterations && !HasFatalFailure() ; ++iter) {
const int ref_stride = rng_(MAX_SB_SIZE + 1);
for (int i = 0 ; i < MAX_SB_SQUARE ; ++i) {
ref[i] = rng_(1<<12);
wsrc[i] = rng_(1<<12) * rng_(kMaskMax * kMaskMax + 1);
mask[i] = rng_(kMaskMax * kMaskMax + 1);
}
const unsigned int ref_res = ref_func_(CONVERT_TO_BYTEPTR(ref), ref_stride,
wsrc, mask);
const unsigned int tst_res = tst_func_(CONVERT_TO_BYTEPTR(ref), ref_stride,
wsrc, mask);
ASSERT_EQ(ref_res, tst_res);
}
}
TEST_P(ObmcSadHBDTest, ExtremeValues) {
DECLARE_ALIGNED(32, uint16_t, ref[MAX_SB_SQUARE]);
DECLARE_ALIGNED(32, int32_t, wsrc[MAX_SB_SQUARE]);
DECLARE_ALIGNED(32, int32_t, mask[MAX_SB_SQUARE]);
for (int iter = 0 ; iter < MAX_SB_SIZE && !HasFatalFailure() ; ++iter) {
const int ref_stride = iter;
for (int i = 0 ; i < MAX_SB_SQUARE ; ++i) {
ref[i] = (1 << 12) - 1;
wsrc[i] = ((1 << 12) - 1) * kMaskMax * kMaskMax;
mask[i] = kMaskMax * kMaskMax;
}
const unsigned int ref_res = ref_func_(CONVERT_TO_BYTEPTR(ref), ref_stride,
wsrc, mask);
const unsigned int tst_res = tst_func_(CONVERT_TO_BYTEPTR(ref), ref_stride,
wsrc, mask);
ASSERT_EQ(ref_res, tst_res);
}
}
#if HAVE_SSE4_1
ObmcSadHBDTest::ParamType sse4_functions_hbd[] = {
#if CONFIG_EXT_PARTITION
make_tuple(vpx_highbd_obmc_sad128x128_c, vpx_highbd_obmc_sad128x128_sse4_1),
make_tuple(vpx_highbd_obmc_sad128x64_c, vpx_highbd_obmc_sad128x64_sse4_1),
make_tuple(vpx_highbd_obmc_sad64x128_c, vpx_highbd_obmc_sad64x128_sse4_1),
#endif // CONFIG_EXT_PARTITION
make_tuple(vpx_highbd_obmc_sad64x64_c, vpx_highbd_obmc_sad64x64_sse4_1),
make_tuple(vpx_highbd_obmc_sad64x32_c, vpx_highbd_obmc_sad64x32_sse4_1),
make_tuple(vpx_highbd_obmc_sad32x64_c, vpx_highbd_obmc_sad32x64_sse4_1),
make_tuple(vpx_highbd_obmc_sad32x32_c, vpx_highbd_obmc_sad32x32_sse4_1),
make_tuple(vpx_highbd_obmc_sad32x16_c, vpx_highbd_obmc_sad32x16_sse4_1),
make_tuple(vpx_highbd_obmc_sad16x32_c, vpx_highbd_obmc_sad16x32_sse4_1),
make_tuple(vpx_highbd_obmc_sad16x16_c, vpx_highbd_obmc_sad16x16_sse4_1),
make_tuple(vpx_highbd_obmc_sad16x8_c, vpx_highbd_obmc_sad16x8_sse4_1),
make_tuple(vpx_highbd_obmc_sad8x16_c, vpx_highbd_obmc_sad8x16_sse4_1),
make_tuple(vpx_highbd_obmc_sad8x8_c, vpx_highbd_obmc_sad8x8_sse4_1),
make_tuple(vpx_highbd_obmc_sad8x4_c, vpx_highbd_obmc_sad8x4_sse4_1),
make_tuple(vpx_highbd_obmc_sad4x8_c, vpx_highbd_obmc_sad4x8_sse4_1),
make_tuple(vpx_highbd_obmc_sad4x4_c, vpx_highbd_obmc_sad4x4_sse4_1)
};
INSTANTIATE_TEST_CASE_P(SSE4_1_C_COMPARE, ObmcSadHBDTest,
::testing::ValuesIn(sse4_functions_hbd));
#endif // HAVE_SSE4_1
#endif // CONFIG_VP9_HIGHBITDEPTH
} // namespace
......@@ -187,6 +187,10 @@ LIBVPX_TEST_SRCS-$(HAVE_SSSE3) += masked_sad_test.cc
LIBVPX_TEST_SRCS-$(CONFIG_VP10_ENCODER) += blend_mask6_test.cc
endif
ifeq ($(CONFIG_OBMC),yes)
LIBVPX_TEST_SRCS-$(CONFIG_VP10_ENCODER) += obmc_sad_test.cc
endif
ifeq ($(CONFIG_VP9_HIGHBITDEPTH),yes)
LIBVPX_TEST_SRCS-$(HAVE_SSE4_1) += vp10_highbd_iht_test.cc
endif # CONFIG_VP9_HIGHBITDEPTH
......
......@@ -314,6 +314,9 @@ ifeq ($(CONFIG_EXT_INTER),yes)
DSP_SRCS-$(HAVE_SSSE3) += x86/masked_sad_intrin_ssse3.c
DSP_SRCS-$(HAVE_SSSE3) += x86/masked_variance_intrin_ssse3.c
endif #CONFIG_EXT_INTER
ifeq ($(CONFIG_OBMC),yes)
DSP_SRCS-$(HAVE_SSE4_1) += x86/obmc_sad_sse4.c
endif #CONFIG_OBMC
endif #CONFIG_VP10_ENCODER
ifeq ($(CONFIG_USE_X86INC),yes)
......
......@@ -1104,14 +1104,14 @@ if (vpx_config("CONFIG_OBMC") eq "yes") {
foreach (@block_sizes) {
($w, $h) = @$_;
add_proto qw/unsigned int/, "vpx_obmc_sad${w}x${h}", "const uint8_t *ref_ptr, int ref_stride, const int32_t *wsrc_ptr, const int32_t *mask";
specialize "vpx_obmc_sad${w}x${h}";
specialize "vpx_obmc_sad${w}x${h}", qw/sse4_1/;
}
if (vpx_config("CONFIG_VP9_HIGHBITDEPTH") eq "yes") {
foreach (@block_sizes) {
($w, $h) = @$_;
add_proto qw/unsigned int/, "vpx_highbd_obmc_sad${w}x${h}", "const uint8_t *ref_ptr, int ref_stride, const int32_t *wsrc_ptr, const int32_t *mask";
specialize "vpx_highbd_obmc_sad${w}x${h}";
specialize "vpx_highbd_obmc_sad${w}x${h}", qw/sse4_1/;
}
}
}
......
/*
* Copyright (c) 2016 The WebM project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include <assert.h>
#include <immintrin.h>
#include "./vpx_config.h"
#include "vpx_ports/mem.h"
#include "vpx/vpx_integer.h"
#include "vpx_dsp/x86/synonyms.h"
////////////////////////////////////////////////////////////////////////////////
// 8 bit
////////////////////////////////////////////////////////////////////////////////
static INLINE unsigned int obmc_sad_w4(const uint8_t *a, const int a_stride,
const int32_t *b, const int32_t *m,
const int height) {
const int a_step = a_stride - 4;
int n = 0;
__m128i v_sad_d = _mm_setzero_si128();
do {
const __m128i v_a_b = xx_loadl_32(a + n);
const __m128i v_m_d = xx_load_128(m + n);
const __m128i v_b_d = xx_load_128(b + n);
const __m128i v_a_d = _mm_cvtepu8_epi32(v_a_b);
// Values in both a and m fit in 15 bits, and are packed at 32 bit
// boundaries. We use pmaddwd, as it has lower latency on Haswell
// than pmulld but produces the same result with these inputs.
const __m128i v_am_d = _mm_madd_epi16(v_a_d, v_m_d);
const __m128i v_diff_d = _mm_sub_epi32(v_b_d, v_am_d);
const __m128i v_absdiff_d = _mm_abs_epi32(v_diff_d);
// Rounded absolute difference
const __m128i v_rad_d = xx_roundn_epu32(v_absdiff_d, 12);
v_sad_d = _mm_add_epi32(v_sad_d, v_rad_d);
n += 4;
if (n % 4 == 0)
a += a_step;
} while (n < 4 * height);
return xx_hsum_epi32_si32(v_sad_d);
}
static INLINE unsigned int obmc_sad_w8n(const uint8_t *a, const int a_stride,
const int32_t *b, const int32_t *m,
const int width, const int height) {
const int a_step = a_stride - width;
int n = 0;
__m128i v_sad_d = _mm_setzero_si128();
assert(width >= 8 && (width & (width - 1)) == 0);
do {
const __m128i v_a1_b = xx_loadl_32(a + n + 4);
const __m128i v_m1_d = xx_load_128(m + n + 4);
const __m128i v_b1_d = xx_load_128(b + n + 4);
const __m128i v_a0_b = xx_loadl_32(a + n);
const __m128i v_m0_d = xx_load_128(m + n);
const __m128i v_b0_d = xx_load_128(b + n);
const __m128i v_a0_d = _mm_cvtepu8_epi32(v_a0_b);
const __m128i v_a1_d = _mm_cvtepu8_epi32(v_a1_b);
// Values in both a and m fit in 15 bits, and are packed at 32 bit
// boundaries. We use pmaddwd, as it has lower latency on Haswell
// than pmulld but produces the same result with these inputs.
const __m128i v_am0_d = _mm_madd_epi16(v_a0_d, v_m0_d);
const __m128i v_am1_d = _mm_madd_epi16(v_a1_d, v_m1_d);
const __m128i v_diff0_d = _mm_sub_epi32(v_b0_d, v_am0_d);
const __m128i v_diff1_d = _mm_sub_epi32(v_b1_d, v_am1_d);
const __m128i v_absdiff0_d = _mm_abs_epi32(v_diff0_d);
const __m128i v_absdiff1_d = _mm_abs_epi32(v_diff1_d);
// Rounded absolute difference
const __m128i v_rad0_d = xx_roundn_epu32(v_absdiff0_d, 12);
const __m128i v_rad1_d = xx_roundn_epu32(v_absdiff1_d, 12);
v_sad_d = _mm_add_epi32(v_sad_d, v_rad0_d);
v_sad_d = _mm_add_epi32(v_sad_d, v_rad1_d);
n += 8;
if (n % width == 0)
a += a_step;
} while (n < width * height);
return xx_hsum_epi32_si32(v_sad_d);
}
#define OBMCSADWXH(w, h) \
unsigned int vpx_obmc_sad##w##x##h##_sse4_1(const uint8_t *ref, \
int ref_stride, \
const int32_t *wsrc, \
const int32_t *msk) { \
if (w == 4) \
return obmc_sad_w4(ref, ref_stride, wsrc, msk, h); \
else \
return obmc_sad_w8n(ref, ref_stride, wsrc, msk, w, h); \
}
#if CONFIG_EXT_PARTITION
OBMCSADWXH(128, 128)
OBMCSADWXH(128, 64)
OBMCSADWXH(64, 128)
#endif // CONFIG_EXT_PARTITION
OBMCSADWXH(64, 64)
OBMCSADWXH(64, 32)
OBMCSADWXH(32, 64)
OBMCSADWXH(32, 32)
OBMCSADWXH(32, 16)
OBMCSADWXH(16, 32)
OBMCSADWXH(16, 16)
OBMCSADWXH(16, 8)
OBMCSADWXH(8, 16)
OBMCSADWXH(8, 8)
OBMCSADWXH(8, 4)
OBMCSADWXH(4, 8)
OBMCSADWXH(4, 4)
////////////////////////////////////////////////////////////////////////////////
// High bit-depth
////////////////////////////////////////////////////////////////////////////////
#if CONFIG_VP9_HIGHBITDEPTH
static INLINE unsigned int hbd_obmc_sad_w4(const uint8_t *a8,
const int a_stride,
const int32_t *b, const int32_t *m,
const int height) {
const uint16_t *a = CONVERT_TO_SHORTPTR(a8);
const int a_step = a_stride - 4;
int n = 0;
__m128i v_sad_d = _mm_setzero_si128();
do {
const __m128i v_a_w = xx_loadl_64(a + n);
const __m128i v_m_d = xx_load_128(m + n);
const __m128i v_b_d = xx_load_128(b + n);
const __m128i v_a_d = _mm_cvtepu16_epi32(v_a_w);
// Values in both a and m fit in 15 bits, and are packed at 32 bit
// boundaries. We use pmaddwd, as it has lower latency on Haswell
// than pmulld but produces the same result with these inputs.
const __m128i v_am_d = _mm_madd_epi16(v_a_d, v_m_d);
const __m128i v_diff_d = _mm_sub_epi32(v_b_d, v_am_d);
const __m128i v_absdiff_d = _mm_abs_epi32(v_diff_d);
// Rounded absolute difference
const __m128i v_rad_d = xx_roundn_epu32(v_absdiff_d, 12);
v_sad_d = _mm_add_epi32(v_sad_d, v_rad_d);
n += 4;
if (n % 4 == 0)
a += a_step;
} while (n < 4 * height);
return xx_hsum_epi32_si32(v_sad_d);
}
static INLINE unsigned int hbd_obmc_sad_w8n(const uint8_t *a8,
const int a_stride,
const int32_t *b, const int32_t *m,
const int width, const int height) {
const uint16_t *a = CONVERT_TO_SHORTPTR(a8);
const int a_step = a_stride - width;
int n = 0;
__m128i v_sad_d = _mm_setzero_si128();
assert(width >= 8 && (width & (width - 1)) == 0);
do {
const __m128i v_a1_w = xx_loadl_64(a + n + 4);
const __m128i v_m1_d = xx_load_128(m + n + 4);
const __m128i v_b1_d = xx_load_128(b + n + 4);
const __m128i v_a0_w = xx_loadl_64(a + n);
const __m128i v_m0_d = xx_load_128(m + n);
const __m128i v_b0_d = xx_load_128(b + n);
const __m128i v_a0_d = _mm_cvtepu16_epi32(v_a0_w);
const __m128i v_a1_d = _mm_cvtepu16_epi32(v_a1_w);
// Values in both a and m fit in 15 bits, and are packed at 32 bit
// boundaries. We use pmaddwd, as it has lower latency on Haswell
// than pmulld but produces the same result with these inputs.
const __m128i v_am0_d = _mm_madd_epi16(v_a0_d, v_m0_d);
const __m128i v_am1_d = _mm_madd_epi16(v_a1_d, v_m1_d);
const __m128i v_diff0_d = _mm_sub_epi32(v_b0_d, v_am0_d);
const __m128i v_diff1_d = _mm_sub_epi32(v_b1_d, v_am1_d);
const __m128i v_absdiff0_d = _mm_abs_epi32(v_diff0_d);
const __m128i v_absdiff1_d = _mm_abs_epi32(v_diff1_d);
// Rounded absolute difference
const __m128i v_rad0_d = xx_roundn_epu32(v_absdiff0_d, 12);
const __m128i v_rad1_d = xx_roundn_epu32(v_absdiff1_d, 12);
v_sad_d = _mm_add_epi32(v_sad_d, v_rad0_d);
v_sad_d = _mm_add_epi32(v_sad_d, v_rad1_d);
n += 8;
if (n % width == 0)
a += a_step;
} while (n < width * height);
return xx_hsum_epi32_si32(v_sad_d);
}
#define HBD_OBMCSADWXH(w, h) \
unsigned int vpx_highbd_obmc_sad##w##x##h##_sse4_1(const uint8_t *ref, \
int ref_stride, \
const int32_t *wsrc, \
const int32_t *msk) { \
if (w == 4) \
return hbd_obmc_sad_w4(ref, ref_stride, wsrc, msk, h); \
else \
return hbd_obmc_sad_w8n(ref, ref_stride, wsrc, msk, w, h); \
}
#if CONFIG_EXT_PARTITION
HBD_OBMCSADWXH(128, 128)
HBD_OBMCSADWXH(128, 64)
HBD_OBMCSADWXH(64, 128)
#endif // CONFIG_EXT_PARTITION
HBD_OBMCSADWXH(64, 64)
HBD_OBMCSADWXH(64, 32)
HBD_OBMCSADWXH(32, 64)
HBD_OBMCSADWXH(32, 32)
HBD_OBMCSADWXH(32, 16)
HBD_OBMCSADWXH(16, 32)
HBD_OBMCSADWXH(16, 16)
HBD_OBMCSADWXH(16, 8)
HBD_OBMCSADWXH(8, 16)
HBD_OBMCSADWXH(8, 8)
HBD_OBMCSADWXH(8, 4)
HBD_OBMCSADWXH(4, 8)
HBD_OBMCSADWXH(4, 4)
#endif // CONFIG_VP9_HIGHBITDEPTH
......@@ -66,4 +66,18 @@ static INLINE __m128i xx_roundn_epu16(__m128i v_val_w, int bits) {
return _mm_avg_epu16(v_s_w, _mm_setzero_si128());
}
static INLINE __m128i xx_roundn_epu32(__m128i v_val_d, int bits) {
const __m128i v_bias_d = _mm_set1_epi32(1 << (bits - 1));
const __m128i v_tmp_d = _mm_add_epi32(v_val_d, v_bias_d);
return _mm_srli_epi32(v_tmp_d, bits);
}
#ifdef __SSSE3__
static INLINE int32_t xx_hsum_epi32_si32(__m128i v_d) {
v_d = _mm_hadd_epi32(v_d, v_d);
v_d = _mm_hadd_epi32(v_d, v_d);
return _mm_cvtsi128_si32(v_d);
}
#endif // __SSSE3__
#endif // VPX_DSP_X86_SYNONYS_H_
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment