Commit 2d44b697 authored by Yi Luo's avatar Yi Luo

Add fast path quantizer AVX2

- Function level improves 36% against sse2.
- Encoder speeds up 2.6% at user level on i7-6700.

Change-Id: I9e43ce60b1e0de8f532249e5c035851463d75dbb
parent 1c602b38
......@@ -121,6 +121,7 @@ AV1_CX_SRCS-yes += encoder/encint.h
endif
AV1_CX_SRCS-$(HAVE_SSE2) += encoder/x86/av1_quantize_sse2.c
AV1_CX_SRCS-$(HAVE_AVX2) += encoder/x86/av1_quantize_avx2.c
AV1_CX_SRCS-$(HAVE_SSE2) += encoder/x86/temporal_filter_apply_sse2.asm
ifeq ($(CONFIG_HIGHBITDEPTH),yes)
AV1_CX_SRCS-$(HAVE_SSE2) += encoder/x86/highbd_block_error_intrin_sse2.c
......
......@@ -320,7 +320,7 @@ if (aom_config("CONFIG_AOM_QM") eq "yes") {
specialize qw/av1_block_error avx2/;
add_proto qw/void av1_quantize_fp/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan";
specialize qw/av1_quantize_fp sse2/;
specialize qw/av1_quantize_fp sse2 avx2/;
add_proto qw/void av1_quantize_fp_32x32/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan";
......@@ -337,7 +337,7 @@ if (aom_config("CONFIG_AOM_QM") eq "yes") {
specialize qw/av1_block_error_fp neon sse2/;
add_proto qw/void av1_quantize_fp/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan";
specialize qw/av1_quantize_fp neon sse2/, "$ssse3_x86_64";
specialize qw/av1_quantize_fp neon sse2 avx2/, "$ssse3_x86_64";
add_proto qw/void av1_quantize_fp_32x32/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, int skip_block, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan";
specialize qw/av1_quantize_fp_32x32/, "$ssse3_x86_64";
......
/*
* Copyright (c) 2017, Alliance for Open Media. All rights reserved
*
* This source code is subject to the terms of the BSD 2 Clause License and
* the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
* was not distributed with this source code in the LICENSE file, you can
* obtain it at www.aomedia.org/license/software. If the Alliance for Open
* Media Patent License 1.0 was not distributed with this source code in the
* PATENTS file, you can obtain it at www.aomedia.org/license/patent.
*/
#include <immintrin.h>
#include "./av1_rtcd.h"
#include "aom/aom_integer.h"
#include "aom_dsp/aom_dsp_common.h"
static INLINE void read_coeff(const tran_low_t *coeff, __m256i *c) {
#if CONFIG_HIGHBITDEPTH
const __m256i x0 = _mm256_loadu_si256((const __m256i *)coeff);
const __m256i x1 = _mm256_loadu_si256((const __m256i *)coeff + 1);
*c = _mm256_packs_epi32(x0, x1);
*c = _mm256_permute4x64_epi64(*c, 0xD8);
#else
*c = _mm256_loadu_si256((const __m256i *)coeff);
#endif
}
static INLINE void write_zero(tran_low_t *qcoeff) {
const __m256i zero = _mm256_setzero_si256();
#if CONFIG_HIGHBITDEPTH
_mm256_storeu_si256((__m256i *)qcoeff, zero);
_mm256_storeu_si256((__m256i *)qcoeff + 1, zero);
#else
_mm256_storeu_si256((__m256i *)qcoeff, zero);
#endif
}
static INLINE void init_one_qp(const __m128i *p, __m256i *qp) {
const __m128i ac = _mm_unpackhi_epi64(*p, *p);
*qp = _mm256_insertf128_si256(_mm256_castsi128_si256(*p), ac, 1);
}
static INLINE void init_qp(const int16_t *round_ptr, const int16_t *quant_ptr,
const int16_t *dequant_ptr, int log_scale,
__m256i *thr, __m256i *qp) {
const __m128i round = _mm_loadu_si128((const __m128i *)round_ptr);
const __m128i quant = _mm_loadu_si128((const __m128i *)quant_ptr);
const __m128i dequant = _mm_loadu_si128((const __m128i *)dequant_ptr);
init_one_qp(&round, &qp[0]);
init_one_qp(&quant, &qp[1]);
init_one_qp(&dequant, &qp[2]);
*thr = _mm256_srai_epi16(qp[2], 1 + log_scale);
}
static INLINE void update_qp(int log_scale, __m256i *thr, __m256i *qp) {
qp[0] = _mm256_permute2x128_si256(qp[0], qp[0], 0x11);
qp[1] = _mm256_permute2x128_si256(qp[1], qp[1], 0x11);
qp[2] = _mm256_permute2x128_si256(qp[2], qp[2], 0x11);
*thr = _mm256_srai_epi16(qp[2], 1 + log_scale);
}
static INLINE void quantize(const __m256i *thr, const __m256i *qp, __m256i *c,
const int16_t *iscan_ptr, tran_low_t *qcoeff,
tran_low_t *dqcoeff, __m256i *eob) {
const __m256i abs = _mm256_abs_epi16(*c);
const __m256i mask = _mm256_cmpgt_epi16(abs, *thr);
const int nzflag = _mm256_movemask_epi8(mask);
if (nzflag) {
__m256i q = _mm256_adds_epi16(abs, qp[0]);
q = _mm256_mulhi_epi16(q, qp[1]);
q = _mm256_sign_epi16(q, *c);
const __m256i dq = _mm256_mullo_epi16(q, qp[2]);
#if CONFIG_HIGHBITDEPTH
__m256i sign_bits = _mm256_srai_epi16(q, 15);
__m256i y0 = _mm256_unpacklo_epi16(q, sign_bits);
__m256i y1 = _mm256_unpackhi_epi16(q, sign_bits);
__m256i x0 = _mm256_permute2x128_si256(y0, y1, 0x20);
__m256i x1 = _mm256_permute2x128_si256(y0, y1, 0x31);
_mm256_storeu_si256((__m256i *)qcoeff, x0);
_mm256_storeu_si256((__m256i *)qcoeff + 1, x1);
sign_bits = _mm256_srai_epi16(dq, 15);
y0 = _mm256_unpacklo_epi16(dq, sign_bits);
y1 = _mm256_unpackhi_epi16(dq, sign_bits);
x0 = _mm256_permute2x128_si256(y0, y1, 0x20);
x1 = _mm256_permute2x128_si256(y0, y1, 0x31);
_mm256_storeu_si256((__m256i *)dqcoeff, x0);
_mm256_storeu_si256((__m256i *)dqcoeff + 1, x1);
#else
_mm256_storeu_si256((__m256i *)qcoeff, q);
_mm256_storeu_si256((__m256i *)dqcoeff, dq);
#endif
const __m256i zero = _mm256_setzero_si256();
const __m256i iscan = _mm256_loadu_si256((const __m256i *)iscan_ptr);
const __m256i zero_coeff = _mm256_cmpeq_epi16(dq, zero);
const __m256i nzero_coeff = _mm256_cmpeq_epi16(zero_coeff, zero);
__m256i cur_eob = _mm256_sub_epi16(iscan, nzero_coeff);
cur_eob = _mm256_and_si256(cur_eob, nzero_coeff);
*eob = _mm256_max_epi16(*eob, cur_eob);
} else {
write_zero(qcoeff);
write_zero(dqcoeff);
}
}
void av1_quantize_fp_avx2(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
int skip_block, const int16_t *zbin_ptr,
const int16_t *round_ptr, const int16_t *quant_ptr,
const int16_t *quant_shift_ptr,
tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr,
const int16_t *dequant_ptr, uint16_t *eob_ptr,
const int16_t *scan_ptr, const int16_t *iscan_ptr) {
(void)scan_ptr;
(void)zbin_ptr;
(void)quant_shift_ptr;
const unsigned int step = 16;
if (LIKELY(!skip_block)) {
__m256i qp[3];
__m256i coeff, thr;
const int log_scale = 0;
init_qp(round_ptr, quant_ptr, dequant_ptr, log_scale, &thr, qp);
read_coeff(coeff_ptr, &coeff);
__m256i eob = _mm256_setzero_si256();
quantize(&thr, qp, &coeff, iscan_ptr, qcoeff_ptr, dqcoeff_ptr, &eob);
coeff_ptr += step;
qcoeff_ptr += step;
dqcoeff_ptr += step;
iscan_ptr += step;
n_coeffs -= step;
update_qp(log_scale, &thr, qp);
while (n_coeffs > 0) {
read_coeff(coeff_ptr, &coeff);
quantize(&thr, qp, &coeff, iscan_ptr, qcoeff_ptr, dqcoeff_ptr, &eob);
coeff_ptr += step;
qcoeff_ptr += step;
dqcoeff_ptr += step;
iscan_ptr += step;
n_coeffs -= step;
}
{
__m256i eob_s;
eob_s = _mm256_shuffle_epi32(eob, 0xe);
eob = _mm256_max_epi16(eob, eob_s);
eob_s = _mm256_shufflelo_epi16(eob, 0xe);
eob = _mm256_max_epi16(eob, eob_s);
eob_s = _mm256_shufflelo_epi16(eob, 1);
eob = _mm256_max_epi16(eob, eob_s);
const __m128i final_eob = _mm_max_epi16(_mm256_castsi256_si128(eob),
_mm256_extractf128_si256(eob, 1));
*eob_ptr = _mm_extract_epi16(final_eob, 0);
}
} else {
do {
write_zero(qcoeff_ptr);
write_zero(dqcoeff_ptr);
qcoeff_ptr += step;
dqcoeff_ptr += step;
n_coeffs -= step;
} while (n_coeffs > 0);
*eob_ptr = 0;
}
}
......@@ -14,6 +14,7 @@
#include "./aom_config.h"
#include "./av1_rtcd.h"
#include "aom/aom_codec.h"
#include "aom_ports/aom_timer.h"
#include "av1/encoder/encoder.h"
#include "av1/encoder/av1_quantize.h"
#include "test/acm_random.h"
......@@ -51,10 +52,10 @@ class QuantizeTest : public ::testing::TestWithParam<QuantizeParam> {
virtual ~QuantizeTest() {}
virtual void SetUp() {
qtab_ = reinterpret_cast<QuanTable *>(aom_memalign(16, sizeof(*qtab_)));
qtab_ = reinterpret_cast<QuanTable *>(aom_memalign(32, sizeof(*qtab_)));
const int n_coeffs = coeff_num();
coeff_ = reinterpret_cast<tran_low_t *>(
aom_memalign(16, 6 * n_coeffs * sizeof(tran_low_t)));
aom_memalign(32, 6 * n_coeffs * sizeof(tran_low_t)));
InitQuantizer();
}
......@@ -91,7 +92,6 @@ class QuantizeTest : public ::testing::TestWithParam<QuantizeParam> {
const int16_t *quant_fp = qtab_->quant.y_quant_fp[q];
const int16_t *quant_shift = qtab_->quant.y_quant_shift[q];
const int16_t *dequant = qtab_->dequant.y_dequant[q];
const size_t bufferSize = n_coeffs;
for (int i = 0; i < test_num; ++i) {
if (is_loop) FillCoeffRandom();
......@@ -106,11 +106,20 @@ class QuantizeTest : public ::testing::TestWithParam<QuantizeParam> {
coeff_ptr, n_coeffs, skip_block, zbin, round_fp, quant_fp,
quant_shift, qcoeff, dqcoeff, dequant, &eob[1], sc->scan, sc->iscan));
ASSERT_NO_FATAL_FAILURE(
CompareResults(qcoeff_ref, qcoeff, bufferSize, "Qcoeff", q, i));
ASSERT_NO_FATAL_FAILURE(
CompareResults(dqcoeff_ref, dqcoeff, bufferSize, "Dqcoeff", q, i));
ASSERT_EQ(eob[0], eob[1]) << "eobs mismatch on test: " << i;
for (int j = 0; j < n_coeffs; ++j) {
ASSERT_EQ(qcoeff_ref[j], qcoeff[j])
<< "Q mismatch on test: " << i << " at position: " << j
<< " Q: " << q << " coeff: " << coeff_ptr[j];
}
for (int j = 0; j < n_coeffs; ++j) {
ASSERT_EQ(dqcoeff_ref[j], dqcoeff[j])
<< "Dq mismatch on test: " << i << " at position: " << j
<< " Q: " << q << " coeff: " << coeff_ptr[j];
}
ASSERT_EQ(eob[0], eob[1]) << "eobs mismatch on test: " << i
<< " Q: " << q;
}
}
......@@ -181,12 +190,12 @@ TEST_P(QuantizeTest, ZeroInput) {
TEST_P(QuantizeTest, LargeNegativeInput) {
FillDcLargeNegative();
QuantizeRun(false);
QuantizeRun(false, 0, 1);
}
TEST_P(QuantizeTest, DcOnlyInput) {
FillDcOnly();
QuantizeRun(false);
QuantizeRun(false, 0, 1);
}
TEST_P(QuantizeTest, RandomInput) { QuantizeRun(true, 0, kTestNum); }
......@@ -197,8 +206,57 @@ TEST_P(QuantizeTest, MultipleQ) {
}
}
TEST_P(QuantizeTest, DISABLED_Speed) {
tran_low_t *coeff_ptr = coeff_;
const intptr_t n_coeffs = coeff_num();
const int skip_block = 0;
tran_low_t *qcoeff_ref = coeff_ptr + n_coeffs;
tran_low_t *dqcoeff_ref = qcoeff_ref + n_coeffs;
tran_low_t *qcoeff = dqcoeff_ref + n_coeffs;
tran_low_t *dqcoeff = qcoeff + n_coeffs;
uint16_t *eob = (uint16_t *)(dqcoeff + n_coeffs);
// Testing uses 2-D DCT scan order table
const SCAN_ORDER *const sc = get_default_scan(tx_size_, DCT_DCT, 0);
// Testing uses luminance quantization table
const int q = 22;
const int16_t *zbin = qtab_->quant.y_zbin[q];
const int16_t *round_fp = qtab_->quant.y_round_fp[q];
const int16_t *quant_fp = qtab_->quant.y_quant_fp[q];
const int16_t *quant_shift = qtab_->quant.y_quant_shift[q];
const int16_t *dequant = qtab_->dequant.y_dequant[q];
const int kNumTests = 5000000;
aom_usec_timer timer;
FillCoeffRandom();
aom_usec_timer_start(&timer);
for (int n = 0; n < kNumTests; ++n) {
quant_(coeff_ptr, n_coeffs, skip_block, zbin, round_fp, quant_fp,
quant_shift, qcoeff, dqcoeff, dequant, eob, sc->scan, sc->iscan);
}
aom_usec_timer_mark(&timer);
const int elapsed_time = static_cast<int>(aom_usec_timer_elapsed(&timer));
printf("Elapsed time: %d us\n", elapsed_time);
}
using std::tr1::make_tuple;
#if HAVE_AVX2
const QuantizeParam kQParamArrayAvx2[] = {
make_tuple(&av1_quantize_fp_c, &av1_quantize_fp_avx2, TX_16X16, AOM_BITS_8),
// make_tuple(&av1_quantize_fp_32x32_c, &av1_quantize_fp_32x32_avx2, TX_32X32,
// AOM_BITS_8)
};
INSTANTIATE_TEST_CASE_P(AVX2, QuantizeTest,
::testing::ValuesIn(kQParamArrayAvx2));
#endif
#if HAVE_SSE2
const QuantizeParam kQParamArraySSE2[] = { make_tuple(
&av1_quantize_fp_c, &av1_quantize_fp_sse2, TX_16X16, AOM_BITS_8) };
......@@ -217,7 +275,8 @@ INSTANTIATE_TEST_CASE_P(SSSE3, QuantizeTest,
// TODO(any):
// The following test does not pass yet
const QuantizeParam kQ32x32ParamArraySSSE3[] = { make_tuple(
av1_quantize_fp_c, av1_quantize_fp_32x32_ssse3, TX_32X32, AOM_BITS_8) };
av1_quantize_fp_32x32_c, av1_quantize_fp_32x32_ssse3, TX_32X32,
AOM_BITS_8) };
INSTANTIATE_TEST_CASE_P(DISABLED_SSSE3, QuantizeTest,
::testing::ValuesIn(kQ32x32ParamArraySSSE3));
#endif
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment