diff --git a/av1/av1_cx.mk b/av1/av1_cx.mk index 6af5c61cb079d8d12b71f79bc7e75f8bb89203ce..4223ad6988561d1a6c33ad749be38f37e63c3c8d 100644 --- a/av1/av1_cx.mk +++ b/av1/av1_cx.mk @@ -121,6 +121,7 @@ AV1_CX_SRCS-yes += encoder/encint.h endif AV1_CX_SRCS-$(HAVE_SSE2) += encoder/x86/av1_quantize_sse2.c +AV1_CX_SRCS-$(HAVE_AVX2) += encoder/x86/av1_quantize_avx2.c AV1_CX_SRCS-$(HAVE_SSE2) += encoder/x86/temporal_filter_apply_sse2.asm ifeq ($(CONFIG_HIGHBITDEPTH),yes) AV1_CX_SRCS-$(HAVE_SSE2) += encoder/x86/highbd_block_error_intrin_sse2.c diff --git a/av1/common/av1_rtcd_defs.pl b/av1/common/av1_rtcd_defs.pl index bd57fdcd335e39ae31ee3ff489f8713af005bd55..ac857cb7c5af0625c1a545144a690405b13ebc6a 100755 --- a/av1/common/av1_rtcd_defs.pl +++ b/av1/common/av1_rtcd_defs.pl @@ -320,7 +320,7 @@ if (aom_config("CONFIG_AOM_QM") eq "yes") { specialize qw/av1_block_error avx2/; add_proto qw/void av1_quantize_fp/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan"; - specialize qw/av1_quantize_fp sse2/; + specialize qw/av1_quantize_fp sse2 avx2/; add_proto qw/void av1_quantize_fp_32x32/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan"; @@ -337,7 +337,7 @@ if (aom_config("CONFIG_AOM_QM") eq "yes") { specialize qw/av1_block_error_fp neon sse2/; add_proto qw/void av1_quantize_fp/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan"; - specialize qw/av1_quantize_fp neon sse2/, "$ssse3_x86_64"; + specialize qw/av1_quantize_fp neon sse2 avx2/, "$ssse3_x86_64"; add_proto qw/void av1_quantize_fp_32x32/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan"; specialize qw/av1_quantize_fp_32x32/, "$ssse3_x86_64"; diff --git a/av1/encoder/x86/av1_quantize_avx2.c b/av1/encoder/x86/av1_quantize_avx2.c new file mode 100644 index 0000000000000000000000000000000000000000..d9302d75a65688f87a7c526d8e9c68dab7cdee9b --- /dev/null +++ b/av1/encoder/x86/av1_quantize_avx2.c @@ -0,0 +1,174 @@ +/* + * Copyright (c) 2017, Alliance for Open Media. All rights reserved + * + * This source code is subject to the terms of the BSD 2 Clause License and + * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License + * was not distributed with this source code in the LICENSE file, you can + * obtain it at www.aomedia.org/license/software. If the Alliance for Open + * Media Patent License 1.0 was not distributed with this source code in the + * PATENTS file, you can obtain it at www.aomedia.org/license/patent. + */ + +#include + +#include "./av1_rtcd.h" +#include "aom/aom_integer.h" +#include "aom_dsp/aom_dsp_common.h" + +static INLINE void read_coeff(const tran_low_t *coeff, __m256i *c) { +#if CONFIG_HIGHBITDEPTH + const __m256i x0 = _mm256_loadu_si256((const __m256i *)coeff); + const __m256i x1 = _mm256_loadu_si256((const __m256i *)coeff + 1); + *c = _mm256_packs_epi32(x0, x1); + *c = _mm256_permute4x64_epi64(*c, 0xD8); +#else + *c = _mm256_loadu_si256((const __m256i *)coeff); +#endif +} + +static INLINE void write_zero(tran_low_t *qcoeff) { + const __m256i zero = _mm256_setzero_si256(); +#if CONFIG_HIGHBITDEPTH + _mm256_storeu_si256((__m256i *)qcoeff, zero); + _mm256_storeu_si256((__m256i *)qcoeff + 1, zero); +#else + _mm256_storeu_si256((__m256i *)qcoeff, zero); +#endif +} + +static INLINE void init_one_qp(const __m128i *p, __m256i *qp) { + const __m128i ac = _mm_unpackhi_epi64(*p, *p); + *qp = _mm256_insertf128_si256(_mm256_castsi128_si256(*p), ac, 1); +} + +static INLINE void init_qp(const int16_t *round_ptr, const int16_t *quant_ptr, + const int16_t *dequant_ptr, int log_scale, + __m256i *thr, __m256i *qp) { + const __m128i round = _mm_loadu_si128((const __m128i *)round_ptr); + const __m128i quant = _mm_loadu_si128((const __m128i *)quant_ptr); + const __m128i dequant = _mm_loadu_si128((const __m128i *)dequant_ptr); + + init_one_qp(&round, &qp[0]); + init_one_qp(&quant, &qp[1]); + init_one_qp(&dequant, &qp[2]); + *thr = _mm256_srai_epi16(qp[2], 1 + log_scale); +} + +static INLINE void update_qp(int log_scale, __m256i *thr, __m256i *qp) { + qp[0] = _mm256_permute2x128_si256(qp[0], qp[0], 0x11); + qp[1] = _mm256_permute2x128_si256(qp[1], qp[1], 0x11); + qp[2] = _mm256_permute2x128_si256(qp[2], qp[2], 0x11); + *thr = _mm256_srai_epi16(qp[2], 1 + log_scale); +} + +static INLINE void quantize(const __m256i *thr, const __m256i *qp, __m256i *c, + const int16_t *iscan_ptr, tran_low_t *qcoeff, + tran_low_t *dqcoeff, __m256i *eob) { + const __m256i abs = _mm256_abs_epi16(*c); + const __m256i mask = _mm256_cmpgt_epi16(abs, *thr); + const int nzflag = _mm256_movemask_epi8(mask); + + if (nzflag) { + __m256i q = _mm256_adds_epi16(abs, qp[0]); + q = _mm256_mulhi_epi16(q, qp[1]); + q = _mm256_sign_epi16(q, *c); + const __m256i dq = _mm256_mullo_epi16(q, qp[2]); + +#if CONFIG_HIGHBITDEPTH + __m256i sign_bits = _mm256_srai_epi16(q, 15); + __m256i y0 = _mm256_unpacklo_epi16(q, sign_bits); + __m256i y1 = _mm256_unpackhi_epi16(q, sign_bits); + __m256i x0 = _mm256_permute2x128_si256(y0, y1, 0x20); + __m256i x1 = _mm256_permute2x128_si256(y0, y1, 0x31); + _mm256_storeu_si256((__m256i *)qcoeff, x0); + _mm256_storeu_si256((__m256i *)qcoeff + 1, x1); + + sign_bits = _mm256_srai_epi16(dq, 15); + y0 = _mm256_unpacklo_epi16(dq, sign_bits); + y1 = _mm256_unpackhi_epi16(dq, sign_bits); + x0 = _mm256_permute2x128_si256(y0, y1, 0x20); + x1 = _mm256_permute2x128_si256(y0, y1, 0x31); + _mm256_storeu_si256((__m256i *)dqcoeff, x0); + _mm256_storeu_si256((__m256i *)dqcoeff + 1, x1); +#else + _mm256_storeu_si256((__m256i *)qcoeff, q); + _mm256_storeu_si256((__m256i *)dqcoeff, dq); +#endif + + const __m256i zero = _mm256_setzero_si256(); + const __m256i iscan = _mm256_loadu_si256((const __m256i *)iscan_ptr); + const __m256i zero_coeff = _mm256_cmpeq_epi16(dq, zero); + const __m256i nzero_coeff = _mm256_cmpeq_epi16(zero_coeff, zero); + __m256i cur_eob = _mm256_sub_epi16(iscan, nzero_coeff); + cur_eob = _mm256_and_si256(cur_eob, nzero_coeff); + *eob = _mm256_max_epi16(*eob, cur_eob); + } else { + write_zero(qcoeff); + write_zero(dqcoeff); + } +} + +void av1_quantize_fp_avx2(const tran_low_t *coeff_ptr, intptr_t n_coeffs, + int skip_block, const int16_t *zbin_ptr, + const int16_t *round_ptr, const int16_t *quant_ptr, + const int16_t *quant_shift_ptr, + tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, + const int16_t *dequant_ptr, uint16_t *eob_ptr, + const int16_t *scan_ptr, const int16_t *iscan_ptr) { + (void)scan_ptr; + (void)zbin_ptr; + (void)quant_shift_ptr; + const unsigned int step = 16; + + if (LIKELY(!skip_block)) { + __m256i qp[3]; + __m256i coeff, thr; + const int log_scale = 0; + + init_qp(round_ptr, quant_ptr, dequant_ptr, log_scale, &thr, qp); + read_coeff(coeff_ptr, &coeff); + + __m256i eob = _mm256_setzero_si256(); + quantize(&thr, qp, &coeff, iscan_ptr, qcoeff_ptr, dqcoeff_ptr, &eob); + + coeff_ptr += step; + qcoeff_ptr += step; + dqcoeff_ptr += step; + iscan_ptr += step; + n_coeffs -= step; + + update_qp(log_scale, &thr, qp); + + while (n_coeffs > 0) { + read_coeff(coeff_ptr, &coeff); + quantize(&thr, qp, &coeff, iscan_ptr, qcoeff_ptr, dqcoeff_ptr, &eob); + + coeff_ptr += step; + qcoeff_ptr += step; + dqcoeff_ptr += step; + iscan_ptr += step; + n_coeffs -= step; + } + { + __m256i eob_s; + eob_s = _mm256_shuffle_epi32(eob, 0xe); + eob = _mm256_max_epi16(eob, eob_s); + eob_s = _mm256_shufflelo_epi16(eob, 0xe); + eob = _mm256_max_epi16(eob, eob_s); + eob_s = _mm256_shufflelo_epi16(eob, 1); + eob = _mm256_max_epi16(eob, eob_s); + const __m128i final_eob = _mm_max_epi16(_mm256_castsi256_si128(eob), + _mm256_extractf128_si256(eob, 1)); + *eob_ptr = _mm_extract_epi16(final_eob, 0); + } + } else { + do { + write_zero(qcoeff_ptr); + write_zero(dqcoeff_ptr); + qcoeff_ptr += step; + dqcoeff_ptr += step; + n_coeffs -= step; + } while (n_coeffs > 0); + *eob_ptr = 0; + } +} diff --git a/test/quantize_func_test.cc b/test/quantize_func_test.cc index d1bfdb22930dfefa3744d9e5dc89dc56f7a11451..c600344061f4a0d40961ded97c9eb174a959c253 100644 --- a/test/quantize_func_test.cc +++ b/test/quantize_func_test.cc @@ -14,6 +14,7 @@ #include "./aom_config.h" #include "./av1_rtcd.h" #include "aom/aom_codec.h" +#include "aom_ports/aom_timer.h" #include "av1/encoder/encoder.h" #include "av1/encoder/av1_quantize.h" #include "test/acm_random.h" @@ -51,10 +52,10 @@ class QuantizeTest : public ::testing::TestWithParam { virtual ~QuantizeTest() {} virtual void SetUp() { - qtab_ = reinterpret_cast(aom_memalign(16, sizeof(*qtab_))); + qtab_ = reinterpret_cast(aom_memalign(32, sizeof(*qtab_))); const int n_coeffs = coeff_num(); coeff_ = reinterpret_cast( - aom_memalign(16, 6 * n_coeffs * sizeof(tran_low_t))); + aom_memalign(32, 6 * n_coeffs * sizeof(tran_low_t))); InitQuantizer(); } @@ -91,7 +92,6 @@ class QuantizeTest : public ::testing::TestWithParam { const int16_t *quant_fp = qtab_->quant.y_quant_fp[q]; const int16_t *quant_shift = qtab_->quant.y_quant_shift[q]; const int16_t *dequant = qtab_->dequant.y_dequant[q]; - const size_t bufferSize = n_coeffs; for (int i = 0; i < test_num; ++i) { if (is_loop) FillCoeffRandom(); @@ -106,11 +106,20 @@ class QuantizeTest : public ::testing::TestWithParam { coeff_ptr, n_coeffs, skip_block, zbin, round_fp, quant_fp, quant_shift, qcoeff, dqcoeff, dequant, &eob[1], sc->scan, sc->iscan)); - ASSERT_NO_FATAL_FAILURE( - CompareResults(qcoeff_ref, qcoeff, bufferSize, "Qcoeff", q, i)); - ASSERT_NO_FATAL_FAILURE( - CompareResults(dqcoeff_ref, dqcoeff, bufferSize, "Dqcoeff", q, i)); - ASSERT_EQ(eob[0], eob[1]) << "eobs mismatch on test: " << i; + for (int j = 0; j < n_coeffs; ++j) { + ASSERT_EQ(qcoeff_ref[j], qcoeff[j]) + << "Q mismatch on test: " << i << " at position: " << j + << " Q: " << q << " coeff: " << coeff_ptr[j]; + } + + for (int j = 0; j < n_coeffs; ++j) { + ASSERT_EQ(dqcoeff_ref[j], dqcoeff[j]) + << "Dq mismatch on test: " << i << " at position: " << j + << " Q: " << q << " coeff: " << coeff_ptr[j]; + } + + ASSERT_EQ(eob[0], eob[1]) << "eobs mismatch on test: " << i + << " Q: " << q; } } @@ -181,12 +190,12 @@ TEST_P(QuantizeTest, ZeroInput) { TEST_P(QuantizeTest, LargeNegativeInput) { FillDcLargeNegative(); - QuantizeRun(false); + QuantizeRun(false, 0, 1); } TEST_P(QuantizeTest, DcOnlyInput) { FillDcOnly(); - QuantizeRun(false); + QuantizeRun(false, 0, 1); } TEST_P(QuantizeTest, RandomInput) { QuantizeRun(true, 0, kTestNum); } @@ -197,8 +206,57 @@ TEST_P(QuantizeTest, MultipleQ) { } } +TEST_P(QuantizeTest, DISABLED_Speed) { + tran_low_t *coeff_ptr = coeff_; + const intptr_t n_coeffs = coeff_num(); + const int skip_block = 0; + + tran_low_t *qcoeff_ref = coeff_ptr + n_coeffs; + tran_low_t *dqcoeff_ref = qcoeff_ref + n_coeffs; + + tran_low_t *qcoeff = dqcoeff_ref + n_coeffs; + tran_low_t *dqcoeff = qcoeff + n_coeffs; + uint16_t *eob = (uint16_t *)(dqcoeff + n_coeffs); + + // Testing uses 2-D DCT scan order table + const SCAN_ORDER *const sc = get_default_scan(tx_size_, DCT_DCT, 0); + + // Testing uses luminance quantization table + const int q = 22; + const int16_t *zbin = qtab_->quant.y_zbin[q]; + const int16_t *round_fp = qtab_->quant.y_round_fp[q]; + const int16_t *quant_fp = qtab_->quant.y_quant_fp[q]; + const int16_t *quant_shift = qtab_->quant.y_quant_shift[q]; + const int16_t *dequant = qtab_->dequant.y_dequant[q]; + const int kNumTests = 5000000; + aom_usec_timer timer; + + FillCoeffRandom(); + + aom_usec_timer_start(&timer); + for (int n = 0; n < kNumTests; ++n) { + quant_(coeff_ptr, n_coeffs, skip_block, zbin, round_fp, quant_fp, + quant_shift, qcoeff, dqcoeff, dequant, eob, sc->scan, sc->iscan); + } + aom_usec_timer_mark(&timer); + + const int elapsed_time = static_cast(aom_usec_timer_elapsed(&timer)); + printf("Elapsed time: %d us\n", elapsed_time); +} + using std::tr1::make_tuple; +#if HAVE_AVX2 +const QuantizeParam kQParamArrayAvx2[] = { + make_tuple(&av1_quantize_fp_c, &av1_quantize_fp_avx2, TX_16X16, AOM_BITS_8), + // make_tuple(&av1_quantize_fp_32x32_c, &av1_quantize_fp_32x32_avx2, TX_32X32, + // AOM_BITS_8) +}; + +INSTANTIATE_TEST_CASE_P(AVX2, QuantizeTest, + ::testing::ValuesIn(kQParamArrayAvx2)); +#endif + #if HAVE_SSE2 const QuantizeParam kQParamArraySSE2[] = { make_tuple( &av1_quantize_fp_c, &av1_quantize_fp_sse2, TX_16X16, AOM_BITS_8) }; @@ -217,7 +275,8 @@ INSTANTIATE_TEST_CASE_P(SSSE3, QuantizeTest, // TODO(any): // The following test does not pass yet const QuantizeParam kQ32x32ParamArraySSSE3[] = { make_tuple( - av1_quantize_fp_c, av1_quantize_fp_32x32_ssse3, TX_32X32, AOM_BITS_8) }; + av1_quantize_fp_32x32_c, av1_quantize_fp_32x32_ssse3, TX_32X32, + AOM_BITS_8) }; INSTANTIATE_TEST_CASE_P(DISABLED_SSSE3, QuantizeTest, ::testing::ValuesIn(kQ32x32ParamArraySSSE3)); #endif