From 23b9b31726dffb4d63e5e27a32c532b73d0b50cd Mon Sep 17 00:00:00 2001 From: Yi Luo Date: Fri, 15 Sep 2017 16:42:41 -0700 Subject: [PATCH] Highbd intra pred H_PRED sse2 optimization sse2 v. C speedup: 4x4 ~8.0x 8x8 ~8.2x 16x16 ~6.5x 32x32 ~3.8x Blocksize: 4x4, 4x8, 8x4, 8x8, 8x16, 16x8, 16x16, 16x32, 32x16, 32x32 Square blocksize code is from libvpx: "30d9a1916 vpxdsp: [x86] add highbd_h_predictor functions", Credit goes to Scott LaVarnway. Speed tests do not support rectangular blocksize yet. Change-Id: I9a1f24aecab8de94f8ea59ec8748fe3537d721ae --- aom_dsp/aom_dsp.cmake | 1 + aom_dsp/aom_dsp.mk | 1 + aom_dsp/aom_dsp_rtcd_defs.pl | 10 ++ aom_dsp/x86/highbd_intrapred_sse2.c | 242 ++++++++++++++++++++++++++++ test/intrapred_test.cc | 15 ++ test/test_intra_pred_speed.cc | 18 +-- 6 files changed, 278 insertions(+), 9 deletions(-) create mode 100644 aom_dsp/x86/highbd_intrapred_sse2.c diff --git a/aom_dsp/aom_dsp.cmake b/aom_dsp/aom_dsp.cmake index b7f9b6b38..0da439299 100644 --- a/aom_dsp/aom_dsp.cmake +++ b/aom_dsp/aom_dsp.cmake @@ -217,6 +217,7 @@ if (CONFIG_HIGHBITDEPTH) set(AOM_DSP_COMMON_INTRIN_SSE2 ${AOM_DSP_COMMON_INTRIN_SSE2} + "${AOM_ROOT}/aom_dsp/x86/highbd_intrapred_sse2.c" "${AOM_ROOT}/aom_dsp/x86/highbd_loopfilter_sse2.c") set(AOM_DSP_COMMON_INTRIN_AVX2 diff --git a/aom_dsp/aom_dsp.mk b/aom_dsp/aom_dsp.mk index 3eb0e3bde..aebeef7a6 100644 --- a/aom_dsp/aom_dsp.mk +++ b/aom_dsp/aom_dsp.mk @@ -78,6 +78,7 @@ DSP_SRCS-$(HAVE_SSSE3) += x86/aom_subpixel_8t_ssse3.asm ifeq ($(CONFIG_HIGHBITDEPTH),yes) DSP_SRCS-$(HAVE_SSE) += x86/highbd_intrapred_sse2.asm DSP_SRCS-$(HAVE_SSE2) += x86/highbd_intrapred_sse2.asm +DSP_SRCS-$(HAVE_SSE2) += x86/highbd_intrapred_sse2.c endif # CONFIG_HIGHBITDEPTH DSP_SRCS-$(HAVE_NEON_ASM) += arm/intrapred_neon_asm$(ASM) diff --git a/aom_dsp/aom_dsp_rtcd_defs.pl b/aom_dsp/aom_dsp_rtcd_defs.pl index 84005848e..ed794fd31 100755 --- a/aom_dsp/aom_dsp_rtcd_defs.pl +++ b/aom_dsp/aom_dsp_rtcd_defs.pl @@ -123,6 +123,16 @@ if (aom_config("CONFIG_HIGHBITDEPTH") eq "yes") { specialize qw/aom_highbd_dc_predictor_16x16 sse2/; specialize qw/aom_highbd_v_predictor_32x32 sse2/; specialize qw/aom_highbd_dc_predictor_32x32 sse2/; + specialize qw/aom_highbd_h_predictor_4x4 sse2/; + specialize qw/aom_highbd_h_predictor_4x8 sse2/; + specialize qw/aom_highbd_h_predictor_8x4 sse2/; + specialize qw/aom_highbd_h_predictor_8x8 sse2/; + specialize qw/aom_highbd_h_predictor_8x16 sse2/; + specialize qw/aom_highbd_h_predictor_16x8 sse2/; + specialize qw/aom_highbd_h_predictor_16x16 sse2/; + specialize qw/aom_highbd_h_predictor_16x32 sse2/; + specialize qw/aom_highbd_h_predictor_32x16 sse2/; + specialize qw/aom_highbd_h_predictor_32x32 sse2/; } # CONFIG_HIGHBITDEPTH # diff --git a/aom_dsp/x86/highbd_intrapred_sse2.c b/aom_dsp/x86/highbd_intrapred_sse2.c new file mode 100644 index 000000000..a2ae2930c --- /dev/null +++ b/aom_dsp/x86/highbd_intrapred_sse2.c @@ -0,0 +1,242 @@ +/* + * Copyright (c) 2017, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include + +#include "./aom_dsp_rtcd.h" + +// ----------------------------------------------------------------------------- +// H_PRED + +void aom_highbd_h_predictor_4x4_sse2(uint16_t *dst, ptrdiff_t stride, + const uint16_t *above, + const uint16_t *left, int bd) { + const __m128i left_u16 = _mm_loadl_epi64((const __m128i *)left); + const __m128i row0 = _mm_shufflelo_epi16(left_u16, 0x0); + const __m128i row1 = _mm_shufflelo_epi16(left_u16, 0x55); + const __m128i row2 = _mm_shufflelo_epi16(left_u16, 0xaa); + const __m128i row3 = _mm_shufflelo_epi16(left_u16, 0xff); + (void)above; + (void)bd; + _mm_storel_epi64((__m128i *)dst, row0); + dst += stride; + _mm_storel_epi64((__m128i *)dst, row1); + dst += stride; + _mm_storel_epi64((__m128i *)dst, row2); + dst += stride; + _mm_storel_epi64((__m128i *)dst, row3); +} + +void aom_highbd_h_predictor_4x8_sse2(uint16_t *dst, ptrdiff_t stride, + const uint16_t *above, + const uint16_t *left, int bd) { + aom_highbd_h_predictor_4x4_sse2(dst, stride, above, left, bd); + dst += stride << 2; + left += 4; + aom_highbd_h_predictor_4x4_sse2(dst, stride, above, left, bd); +} + +void aom_highbd_h_predictor_8x4_sse2(uint16_t *dst, ptrdiff_t stride, + const uint16_t *above, + const uint16_t *left, int bd) { + const __m128i left_u16 = _mm_load_si128((const __m128i *)left); + const __m128i row0 = _mm_shufflelo_epi16(left_u16, 0x0); + const __m128i row1 = _mm_shufflelo_epi16(left_u16, 0x55); + const __m128i row2 = _mm_shufflelo_epi16(left_u16, 0xaa); + const __m128i row3 = _mm_shufflelo_epi16(left_u16, 0xff); + (void)above; + (void)bd; + _mm_store_si128((__m128i *)dst, _mm_unpacklo_epi64(row0, row0)); + dst += stride; + _mm_store_si128((__m128i *)dst, _mm_unpacklo_epi64(row1, row1)); + dst += stride; + _mm_store_si128((__m128i *)dst, _mm_unpacklo_epi64(row2, row2)); + dst += stride; + _mm_store_si128((__m128i *)dst, _mm_unpacklo_epi64(row3, row3)); +} + +void aom_highbd_h_predictor_8x8_sse2(uint16_t *dst, ptrdiff_t stride, + const uint16_t *above, + const uint16_t *left, int bd) { + const __m128i left_u16 = _mm_load_si128((const __m128i *)left); + const __m128i row0 = _mm_shufflelo_epi16(left_u16, 0x0); + const __m128i row1 = _mm_shufflelo_epi16(left_u16, 0x55); + const __m128i row2 = _mm_shufflelo_epi16(left_u16, 0xaa); + const __m128i row3 = _mm_shufflelo_epi16(left_u16, 0xff); + const __m128i row4 = _mm_shufflehi_epi16(left_u16, 0x0); + const __m128i row5 = _mm_shufflehi_epi16(left_u16, 0x55); + const __m128i row6 = _mm_shufflehi_epi16(left_u16, 0xaa); + const __m128i row7 = _mm_shufflehi_epi16(left_u16, 0xff); + (void)above; + (void)bd; + _mm_store_si128((__m128i *)dst, _mm_unpacklo_epi64(row0, row0)); + dst += stride; + _mm_store_si128((__m128i *)dst, _mm_unpacklo_epi64(row1, row1)); + dst += stride; + _mm_store_si128((__m128i *)dst, _mm_unpacklo_epi64(row2, row2)); + dst += stride; + _mm_store_si128((__m128i *)dst, _mm_unpacklo_epi64(row3, row3)); + dst += stride; + _mm_store_si128((__m128i *)dst, _mm_unpackhi_epi64(row4, row4)); + dst += stride; + _mm_store_si128((__m128i *)dst, _mm_unpackhi_epi64(row5, row5)); + dst += stride; + _mm_store_si128((__m128i *)dst, _mm_unpackhi_epi64(row6, row6)); + dst += stride; + _mm_store_si128((__m128i *)dst, _mm_unpackhi_epi64(row7, row7)); +} + +void aom_highbd_h_predictor_8x16_sse2(uint16_t *dst, ptrdiff_t stride, + const uint16_t *above, + const uint16_t *left, int bd) { + aom_highbd_h_predictor_8x8_sse2(dst, stride, above, left, bd); + dst += stride << 3; + left += 8; + aom_highbd_h_predictor_8x8_sse2(dst, stride, above, left, bd); +} + +static INLINE void h_store_16_unpacklo(uint16_t **dst, const ptrdiff_t stride, + const __m128i *row) { + const __m128i val = _mm_unpacklo_epi64(*row, *row); + _mm_store_si128((__m128i *)*dst, val); + _mm_store_si128((__m128i *)(*dst + 8), val); + *dst += stride; +} + +static INLINE void h_store_16_unpackhi(uint16_t **dst, const ptrdiff_t stride, + const __m128i *row) { + const __m128i val = _mm_unpackhi_epi64(*row, *row); + _mm_store_si128((__m128i *)(*dst), val); + _mm_store_si128((__m128i *)(*dst + 8), val); + *dst += stride; +} + +static INLINE void h_predictor_16x8(uint16_t *dst, ptrdiff_t stride, + const uint16_t *left) { + const __m128i left_u16 = _mm_load_si128((const __m128i *)left); + const __m128i row0 = _mm_shufflelo_epi16(left_u16, 0x0); + const __m128i row1 = _mm_shufflelo_epi16(left_u16, 0x55); + const __m128i row2 = _mm_shufflelo_epi16(left_u16, 0xaa); + const __m128i row3 = _mm_shufflelo_epi16(left_u16, 0xff); + const __m128i row4 = _mm_shufflehi_epi16(left_u16, 0x0); + const __m128i row5 = _mm_shufflehi_epi16(left_u16, 0x55); + const __m128i row6 = _mm_shufflehi_epi16(left_u16, 0xaa); + const __m128i row7 = _mm_shufflehi_epi16(left_u16, 0xff); + h_store_16_unpacklo(&dst, stride, &row0); + h_store_16_unpacklo(&dst, stride, &row1); + h_store_16_unpacklo(&dst, stride, &row2); + h_store_16_unpacklo(&dst, stride, &row3); + h_store_16_unpackhi(&dst, stride, &row4); + h_store_16_unpackhi(&dst, stride, &row5); + h_store_16_unpackhi(&dst, stride, &row6); + h_store_16_unpackhi(&dst, stride, &row7); +} + +void aom_highbd_h_predictor_16x8_sse2(uint16_t *dst, ptrdiff_t stride, + const uint16_t *above, + const uint16_t *left, int bd) { + (void)above; + (void)bd; + h_predictor_16x8(dst, stride, left); +} + +void aom_highbd_h_predictor_16x16_sse2(uint16_t *dst, ptrdiff_t stride, + const uint16_t *above, + const uint16_t *left, int bd) { + int i; + (void)above; + (void)bd; + + for (i = 0; i < 2; i++, left += 8) { + h_predictor_16x8(dst, stride, left); + dst += stride << 3; + } +} + +void aom_highbd_h_predictor_16x32_sse2(uint16_t *dst, ptrdiff_t stride, + const uint16_t *above, + const uint16_t *left, int bd) { + int i; + (void)above; + (void)bd; + + for (i = 0; i < 4; i++, left += 8) { + h_predictor_16x8(dst, stride, left); + dst += stride << 3; + } +} + +static INLINE void h_store_32_unpacklo(uint16_t **dst, const ptrdiff_t stride, + const __m128i *row) { + const __m128i val = _mm_unpacklo_epi64(*row, *row); + _mm_store_si128((__m128i *)(*dst), val); + _mm_store_si128((__m128i *)(*dst + 8), val); + _mm_store_si128((__m128i *)(*dst + 16), val); + _mm_store_si128((__m128i *)(*dst + 24), val); + *dst += stride; +} + +static INLINE void h_store_32_unpackhi(uint16_t **dst, const ptrdiff_t stride, + const __m128i *row) { + const __m128i val = _mm_unpackhi_epi64(*row, *row); + _mm_store_si128((__m128i *)(*dst), val); + _mm_store_si128((__m128i *)(*dst + 8), val); + _mm_store_si128((__m128i *)(*dst + 16), val); + _mm_store_si128((__m128i *)(*dst + 24), val); + *dst += stride; +} + +static INLINE void h_predictor_32x8(uint16_t *dst, ptrdiff_t stride, + const uint16_t *left) { + const __m128i left_u16 = _mm_load_si128((const __m128i *)left); + const __m128i row0 = _mm_shufflelo_epi16(left_u16, 0x0); + const __m128i row1 = _mm_shufflelo_epi16(left_u16, 0x55); + const __m128i row2 = _mm_shufflelo_epi16(left_u16, 0xaa); + const __m128i row3 = _mm_shufflelo_epi16(left_u16, 0xff); + const __m128i row4 = _mm_shufflehi_epi16(left_u16, 0x0); + const __m128i row5 = _mm_shufflehi_epi16(left_u16, 0x55); + const __m128i row6 = _mm_shufflehi_epi16(left_u16, 0xaa); + const __m128i row7 = _mm_shufflehi_epi16(left_u16, 0xff); + h_store_32_unpacklo(&dst, stride, &row0); + h_store_32_unpacklo(&dst, stride, &row1); + h_store_32_unpacklo(&dst, stride, &row2); + h_store_32_unpacklo(&dst, stride, &row3); + h_store_32_unpackhi(&dst, stride, &row4); + h_store_32_unpackhi(&dst, stride, &row5); + h_store_32_unpackhi(&dst, stride, &row6); + h_store_32_unpackhi(&dst, stride, &row7); +} + +void aom_highbd_h_predictor_32x16_sse2(uint16_t *dst, ptrdiff_t stride, + const uint16_t *above, + const uint16_t *left, int bd) { + int i; + (void)above; + (void)bd; + + for (i = 0; i < 2; i++, left += 8) { + h_predictor_32x8(dst, stride, left); + dst += stride << 3; + } +} + +void aom_highbd_h_predictor_32x32_sse2(uint16_t *dst, ptrdiff_t stride, + const uint16_t *above, + const uint16_t *left, int bd) { + int i; + (void)above; + (void)bd; + + for (i = 0; i < 4; i++, left += 8) { + h_predictor_32x8(dst, stride, left); + dst += stride << 3; + } +} diff --git a/test/intrapred_test.cc b/test/intrapred_test.cc index e8e87c4b7..06b1e9562 100644 --- a/test/intrapred_test.cc +++ b/test/intrapred_test.cc @@ -140,6 +140,11 @@ const IntraPredFunc IntraPredTestVector8[] = { highbd_entry(dc, 16, 16, sse2, 8), highbd_entry(dc, 32, 32, sse2, 8), highbd_entry(v, 4, 4, sse2, 8), highbd_entry(v, 8, 8, sse2, 8), highbd_entry(v, 16, 16, sse2, 8), highbd_entry(v, 32, 32, sse2, 8), + highbd_entry(h, 4, 4, sse2, 8), highbd_entry(h, 4, 8, sse2, 8), + highbd_entry(h, 8, 4, sse2, 8), highbd_entry(h, 8, 8, sse2, 8), + highbd_entry(h, 8, 16, sse2, 8), highbd_entry(h, 16, 8, sse2, 8), + highbd_entry(h, 16, 16, sse2, 8), highbd_entry(h, 16, 32, sse2, 8), + highbd_entry(h, 32, 16, sse2, 8), highbd_entry(h, 32, 32, sse2, 8), }; INSTANTIATE_TEST_CASE_P(SSE2_TO_C_8, AV1IntraPredTest, @@ -150,6 +155,11 @@ const IntraPredFunc IntraPredTestVector10[] = { highbd_entry(dc, 16, 16, sse2, 10), highbd_entry(dc, 32, 32, sse2, 10), highbd_entry(v, 4, 4, sse2, 10), highbd_entry(v, 8, 8, sse2, 10), highbd_entry(v, 16, 16, sse2, 10), highbd_entry(v, 32, 32, sse2, 10), + highbd_entry(h, 4, 4, sse2, 10), highbd_entry(h, 4, 8, sse2, 10), + highbd_entry(h, 8, 4, sse2, 10), highbd_entry(h, 8, 8, sse2, 10), + highbd_entry(h, 8, 16, sse2, 10), highbd_entry(h, 16, 8, sse2, 10), + highbd_entry(h, 16, 16, sse2, 10), highbd_entry(h, 16, 32, sse2, 10), + highbd_entry(h, 32, 16, sse2, 10), highbd_entry(h, 32, 32, sse2, 10), }; INSTANTIATE_TEST_CASE_P(SSE2_TO_C_10, AV1IntraPredTest, @@ -160,6 +170,11 @@ const IntraPredFunc IntraPredTestVector12[] = { highbd_entry(dc, 16, 16, sse2, 12), highbd_entry(dc, 32, 32, sse2, 12), highbd_entry(v, 4, 4, sse2, 12), highbd_entry(v, 8, 8, sse2, 12), highbd_entry(v, 16, 16, sse2, 12), highbd_entry(v, 32, 32, sse2, 12), + highbd_entry(h, 4, 4, sse2, 12), highbd_entry(h, 4, 8, sse2, 12), + highbd_entry(h, 8, 4, sse2, 12), highbd_entry(h, 8, 8, sse2, 12), + highbd_entry(h, 8, 16, sse2, 12), highbd_entry(h, 16, 8, sse2, 12), + highbd_entry(h, 16, 16, sse2, 12), highbd_entry(h, 16, 32, sse2, 12), + highbd_entry(h, 32, 16, sse2, 12), highbd_entry(h, 32, 32, sse2, 12), }; INSTANTIATE_TEST_CASE_P(SSE2_TO_C_12, AV1IntraPredTest, diff --git a/test/test_intra_pred_speed.cc b/test/test_intra_pred_speed.cc index 73d17458e..9525a62c0 100644 --- a/test/test_intra_pred_speed.cc +++ b/test/test_intra_pred_speed.cc @@ -76,7 +76,7 @@ void CheckMd5Signature(const char name[], const char *const signatures[], int idx) { libaom_test::MD5 md5; md5.Add(reinterpret_cast(data), data_size); - printf("Mode %s[%12s]: %5d ms MD5: %s\n", name, kAv1IntraPredNames[idx], + printf("Mode %s[%13s]: %5d ms MD5: %s\n", name, kAv1IntraPredNames[idx], elapsed_time, md5.Get()); EXPECT_STREQ(signatures[idx], md5.Get()); } @@ -634,8 +634,8 @@ HIGHBD_INTRA_PRED_TEST( #if HAVE_SSE2 HIGHBD_INTRA_PRED_TEST(SSE2, TestHighbdIntraPred4, NULL, NULL, NULL, NULL, NULL, - NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, - NULL, NULL) + aom_highbd_h_predictor_4x4_sse2, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL) #endif // ----------------------------------------------------------------------------- @@ -663,8 +663,8 @@ HIGHBD_INTRA_PRED_TEST( #if HAVE_SSE2 HIGHBD_INTRA_PRED_TEST(SSE2, TestHighbdIntraPred8, NULL, NULL, NULL, NULL, NULL, - NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, - NULL, NULL) + aom_highbd_h_predictor_8x8_sse2, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL) #endif // ----------------------------------------------------------------------------- @@ -692,8 +692,8 @@ HIGHBD_INTRA_PRED_TEST( #if HAVE_SSE2 HIGHBD_INTRA_PRED_TEST(SSE2, TestHighbdIntraPred16, NULL, NULL, NULL, NULL, - NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, - NULL, NULL, NULL) + NULL, aom_highbd_h_predictor_16x16_sse2, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL) #endif // ----------------------------------------------------------------------------- @@ -721,8 +721,8 @@ HIGHBD_INTRA_PRED_TEST( #if HAVE_SSE2 HIGHBD_INTRA_PRED_TEST(SSE2, TestHighbdIntraPred32, NULL, NULL, NULL, NULL, - NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, - NULL, NULL, NULL) + NULL, aom_highbd_h_predictor_32x32_sse2, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL) #endif #endif // CONFIG_HIGHBITDEPTH -- GitLab