Commit f6176abb authored by Yi Luo's avatar Yi Luo
Browse files

Partial IDCT 16x16 avx2

- Function level improvement:
functions      sse2  avx2  percentage
idct16x16_256  365   226   38%
idct16x16_38   n/a   136   n/a
idct16x16_10   171   110   35%
idct16x16_1     34    26   23%

- Integrated in AV1 for default scan order.

Change-Id: Ieb1a8e730bea9c371ebc0e5f4a748640d8f5e921
parent e6ca8e83
...@@ -62,8 +62,10 @@ set(AOM_DSP_COMMON_INTRIN_SSE4_1 ...@@ -62,8 +62,10 @@ set(AOM_DSP_COMMON_INTRIN_SSE4_1
set(AOM_DSP_COMMON_INTRIN_AVX2 set(AOM_DSP_COMMON_INTRIN_AVX2
"${AOM_ROOT}/aom_dsp/x86/aom_subpixel_8t_intrin_avx2.c" "${AOM_ROOT}/aom_dsp/x86/aom_subpixel_8t_intrin_avx2.c"
"${AOM_ROOT}/aom_dsp/x86/fwd_txfm_avx2.c" "${AOM_ROOT}/aom_dsp/x86/loopfilter_avx2.c"
"${AOM_ROOT}/aom_dsp/x86/loopfilter_avx2.c") "${AOM_ROOT}/aom_dsp/x86/inv_txfm_avx2.c"
"${AOM_ROOT}/aom_dsp/x86/inv_txfm_common_avx2.h"
"${AOM_ROOT}/aom_dsp/x86/txfm_common_avx2.h")
set(AOM_DSP_COMMON_ASM_NEON set(AOM_DSP_COMMON_ASM_NEON
"${AOM_ROOT}/aom_dsp/arm/aom_convolve8_avg_neon_asm.asm" "${AOM_ROOT}/aom_dsp/arm/aom_convolve8_avg_neon_asm.asm"
...@@ -282,6 +284,9 @@ if (CONFIG_ENCODERS) ...@@ -282,6 +284,9 @@ if (CONFIG_ENCODERS)
set(AOM_DSP_ENCODER_ASM_SSE4_1 "${AOM_ROOT}/aom_dsp/x86/sad_sse4.asm") set(AOM_DSP_ENCODER_ASM_SSE4_1 "${AOM_ROOT}/aom_dsp/x86/sad_sse4.asm")
set(AOM_DSP_ENCODER_INTRIN_AVX2 set(AOM_DSP_ENCODER_INTRIN_AVX2
"${AOM_ROOT}/aom_dsp/x86/fwd_dct32x32_impl_avx2.h"
"${AOM_ROOT}/aom_dsp/x86/fwd_txfm_avx2.c"
"${AOM_ROOT}/aom_dsp/x86/fwd_txfm_avx2.h"
"${AOM_ROOT}/aom_dsp/x86/sad4d_avx2.c" "${AOM_ROOT}/aom_dsp/x86/sad4d_avx2.c"
"${AOM_ROOT}/aom_dsp/x86/sad_avx2.c" "${AOM_ROOT}/aom_dsp/x86/sad_avx2.c"
"${AOM_ROOT}/aom_dsp/x86/sad_impl_avx2.c" "${AOM_ROOT}/aom_dsp/x86/sad_impl_avx2.c"
......
...@@ -240,6 +240,8 @@ DSP_SRCS-$(HAVE_SSE2) += x86/inv_txfm_sse2.h ...@@ -240,6 +240,8 @@ DSP_SRCS-$(HAVE_SSE2) += x86/inv_txfm_sse2.h
DSP_SRCS-$(HAVE_SSE2) += x86/inv_txfm_sse2.c DSP_SRCS-$(HAVE_SSE2) += x86/inv_txfm_sse2.c
DSP_SRCS-$(HAVE_SSE2) += x86/inv_wht_sse2.asm DSP_SRCS-$(HAVE_SSE2) += x86/inv_wht_sse2.asm
DSP_SRCS-$(HAVE_SSSE3) += x86/inv_txfm_ssse3.c DSP_SRCS-$(HAVE_SSSE3) += x86/inv_txfm_ssse3.c
DSP_SRCS-$(HAVE_AVX2) += x86/inv_txfm_common_avx2.h
DSP_SRCS-$(HAVE_AVX2) += x86/inv_txfm_avx2.c
ifeq ($(HAVE_NEON_ASM),yes) ifeq ($(HAVE_NEON_ASM),yes)
DSP_SRCS-yes += arm/save_reg_neon$(ASM) DSP_SRCS-yes += arm/save_reg_neon$(ASM)
......
...@@ -437,15 +437,16 @@ if (aom_config("CONFIG_HIGHBITDEPTH") eq "yes") { ...@@ -437,15 +437,16 @@ if (aom_config("CONFIG_HIGHBITDEPTH") eq "yes") {
specialize qw/aom_idct8x8_1_add sse2/; specialize qw/aom_idct8x8_1_add sse2/;
add_proto qw/void aom_idct16x16_256_add/, "const tran_low_t *input, uint8_t *dest, int dest_stride"; add_proto qw/void aom_idct16x16_256_add/, "const tran_low_t *input, uint8_t *dest, int dest_stride";
specialize qw/aom_idct16x16_256_add sse2/; specialize qw/aom_idct16x16_256_add sse2 avx2/;
add_proto qw/void aom_idct16x16_38_add/, "const tran_low_t *input, uint8_t *dest, int dest_stride"; add_proto qw/void aom_idct16x16_38_add/, "const tran_low_t *input, uint8_t *dest, int dest_stride";
specialize qw/aom_idct16x16_38_add avx2/;
add_proto qw/void aom_idct16x16_10_add/, "const tran_low_t *input, uint8_t *dest, int dest_stride"; add_proto qw/void aom_idct16x16_10_add/, "const tran_low_t *input, uint8_t *dest, int dest_stride";
specialize qw/aom_idct16x16_10_add sse2/; specialize qw/aom_idct16x16_10_add sse2 avx2/;
add_proto qw/void aom_idct16x16_1_add/, "const tran_low_t *input, uint8_t *dest, int dest_stride"; add_proto qw/void aom_idct16x16_1_add/, "const tran_low_t *input, uint8_t *dest, int dest_stride";
specialize qw/aom_idct16x16_1_add sse2/; specialize qw/aom_idct16x16_1_add sse2 avx2/;
add_proto qw/void aom_idct32x32_1024_add/, "const tran_low_t *input, uint8_t *dest, int dest_stride"; add_proto qw/void aom_idct32x32_1024_add/, "const tran_low_t *input, uint8_t *dest, int dest_stride";
specialize qw/aom_idct32x32_1024_add sse2 ssse3/; specialize qw/aom_idct32x32_1024_add sse2 ssse3/;
...@@ -482,15 +483,16 @@ if (aom_config("CONFIG_HIGHBITDEPTH") eq "yes") { ...@@ -482,15 +483,16 @@ if (aom_config("CONFIG_HIGHBITDEPTH") eq "yes") {
specialize qw/aom_idct8x8_12_add sse2 ssse3 neon dspr2 msa/; specialize qw/aom_idct8x8_12_add sse2 ssse3 neon dspr2 msa/;
add_proto qw/void aom_idct16x16_1_add/, "const tran_low_t *input, uint8_t *dest, int dest_stride"; add_proto qw/void aom_idct16x16_1_add/, "const tran_low_t *input, uint8_t *dest, int dest_stride";
specialize qw/aom_idct16x16_1_add sse2 neon dspr2 msa/; specialize qw/aom_idct16x16_1_add sse2 avx2 neon dspr2 msa/;
add_proto qw/void aom_idct16x16_256_add/, "const tran_low_t *input, uint8_t *dest, int dest_stride"; add_proto qw/void aom_idct16x16_256_add/, "const tran_low_t *input, uint8_t *dest, int dest_stride";
specialize qw/aom_idct16x16_256_add sse2 neon dspr2 msa/; specialize qw/aom_idct16x16_256_add sse2 avx2 neon dspr2 msa/;
add_proto qw/void aom_idct16x16_38_add/, "const tran_low_t *input, uint8_t *dest, int dest_stride"; add_proto qw/void aom_idct16x16_38_add/, "const tran_low_t *input, uint8_t *dest, int dest_stride";
specialize qw/aom_idct16x16_38_add avx2/;
add_proto qw/void aom_idct16x16_10_add/, "const tran_low_t *input, uint8_t *dest, int dest_stride"; add_proto qw/void aom_idct16x16_10_add/, "const tran_low_t *input, uint8_t *dest, int dest_stride";
specialize qw/aom_idct16x16_10_add sse2 neon dspr2 msa/; specialize qw/aom_idct16x16_10_add sse2 avx2 neon dspr2 msa/;
add_proto qw/void aom_idct32x32_1024_add/, "const tran_low_t *input, uint8_t *dest, int dest_stride"; add_proto qw/void aom_idct32x32_1024_add/, "const tran_low_t *input, uint8_t *dest, int dest_stride";
specialize qw/aom_idct32x32_1024_add sse2 ssse3 neon dspr2 msa/; specialize qw/aom_idct32x32_1024_add sse2 ssse3 neon dspr2 msa/;
......
/*
* Copyright (c) 2017, Alliance for Open Media. All rights reserved
*
* This source code is subject to the terms of the BSD 2 Clause License and
* the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
* was not distributed with this source code in the LICENSE file, you can
* obtain it at www.aomedia.org/license/software. If the Alliance for Open
* Media Patent License 1.0 was not distributed with this source code in the
* PATENTS file, you can obtain it at www.aomedia.org/license/patent.
*/
#include <immintrin.h>
#include "./aom_dsp_rtcd.h"
#include "aom_dsp/inv_txfm.h"
#include "aom_dsp/x86/inv_txfm_common_avx2.h"
#include "aom_dsp/x86/txfm_common_avx2.h"
void aom_idct16x16_256_add_avx2(const tran_low_t *input, uint8_t *dest,
int stride) {
__m256i in[16];
load_buffer_16x16(input, in);
mm256_transpose_16x16(in);
av1_idct16_avx2(in);
mm256_transpose_16x16(in);
av1_idct16_avx2(in);
write_buffer_16x16(in, stride, dest);
}
static INLINE void transpose_col_to_row_nz4x4(__m256i *in /*in[4]*/) {
const __m256i u0 = _mm256_unpacklo_epi16(in[0], in[1]);
const __m256i u1 = _mm256_unpacklo_epi16(in[2], in[3]);
const __m256i v0 = _mm256_unpacklo_epi32(u0, u1);
const __m256i v1 = _mm256_unpackhi_epi32(u0, u1);
in[0] = _mm256_permute4x64_epi64(v0, 0xA8);
in[1] = _mm256_permute4x64_epi64(v0, 0xA9);
in[2] = _mm256_permute4x64_epi64(v1, 0xA8);
in[3] = _mm256_permute4x64_epi64(v1, 0xA9);
}
#define SHUFFLE_EPI64(x0, x1, imm8) \
(__m256i) _mm256_shuffle_pd((__m256d)x0, (__m256d)x1, imm8)
static INLINE void transpose_col_to_row_nz4x16(__m256i *in /*in[16]*/) {
int i;
for (i = 0; i < 16; i += 4) {
transpose_col_to_row_nz4x4(&in[i]);
}
for (i = 0; i < 4; ++i) {
in[i] = SHUFFLE_EPI64(in[i], in[i + 4], 0);
in[i + 8] = SHUFFLE_EPI64(in[i + 8], in[i + 12], 0);
}
for (i = 0; i < 4; ++i) {
in[i] = _mm256_permute2x128_si256(in[i], in[i + 8], 0x20);
}
}
// Coefficients 0-7 before the final butterfly
static INLINE void idct16_10_first_half(const __m256i *in, __m256i *out) {
const __m256i c2p28 = pair256_set_epi16(2 * cospi_28_64, 2 * cospi_28_64);
const __m256i c2p04 = pair256_set_epi16(2 * cospi_4_64, 2 * cospi_4_64);
const __m256i v4 = _mm256_mulhrs_epi16(in[2], c2p28);
const __m256i v7 = _mm256_mulhrs_epi16(in[2], c2p04);
const __m256i c2p16 = pair256_set_epi16(2 * cospi_16_64, 2 * cospi_16_64);
const __m256i v0 = _mm256_mulhrs_epi16(in[0], c2p16);
const __m256i v1 = v0;
const __m256i cospi_p16_p16 = _mm256_set1_epi16((int16_t)cospi_16_64);
const __m256i cospi_p16_m16 = pair256_set_epi16(cospi_16_64, -cospi_16_64);
__m256i v5, v6;
unpack_butter_fly(&v7, &v4, &cospi_p16_m16, &cospi_p16_p16, &v5, &v6);
out[0] = _mm256_add_epi16(v0, v7);
out[1] = _mm256_add_epi16(v1, v6);
out[2] = _mm256_add_epi16(v1, v5);
out[3] = _mm256_add_epi16(v0, v4);
out[4] = _mm256_sub_epi16(v0, v4);
out[5] = _mm256_sub_epi16(v1, v5);
out[6] = _mm256_sub_epi16(v1, v6);
out[7] = _mm256_sub_epi16(v0, v7);
}
// Coefficients 8-15 before the final butterfly
static INLINE void idct16_10_second_half(const __m256i *in, __m256i *out) {
const __m256i c2p30 = pair256_set_epi16(2 * cospi_30_64, 2 * cospi_30_64);
const __m256i c2p02 = pair256_set_epi16(2 * cospi_2_64, 2 * cospi_2_64);
const __m256i t0 = _mm256_mulhrs_epi16(in[1], c2p30);
const __m256i t7 = _mm256_mulhrs_epi16(in[1], c2p02);
const __m256i c2m26 = pair256_set_epi16(-2 * cospi_26_64, -2 * cospi_26_64);
const __m256i c2p06 = pair256_set_epi16(2 * cospi_6_64, 2 * cospi_6_64);
const __m256i t3 = _mm256_mulhrs_epi16(in[3], c2m26);
const __m256i t4 = _mm256_mulhrs_epi16(in[3], c2p06);
const __m256i cospi_m08_p24 = pair256_set_epi16(-cospi_8_64, cospi_24_64);
const __m256i cospi_p24_p08 = pair256_set_epi16(cospi_24_64, cospi_8_64);
const __m256i cospi_m24_m08 = pair256_set_epi16(-cospi_24_64, -cospi_8_64);
__m256i t1, t2, t5, t6;
unpack_butter_fly(&t0, &t7, &cospi_m08_p24, &cospi_p24_p08, &t1, &t6);
unpack_butter_fly(&t3, &t4, &cospi_m24_m08, &cospi_m08_p24, &t2, &t5);
out[0] = _mm256_add_epi16(t0, t3);
out[1] = _mm256_add_epi16(t1, t2);
out[6] = _mm256_add_epi16(t6, t5);
out[7] = _mm256_add_epi16(t7, t4);
const __m256i v2 = _mm256_sub_epi16(t1, t2);
const __m256i v3 = _mm256_sub_epi16(t0, t3);
const __m256i v4 = _mm256_sub_epi16(t7, t4);
const __m256i v5 = _mm256_sub_epi16(t6, t5);
const __m256i cospi_p16_p16 = _mm256_set1_epi16((int16_t)cospi_16_64);
const __m256i cospi_p16_m16 = pair256_set_epi16(cospi_16_64, -cospi_16_64);
unpack_butter_fly(&v5, &v2, &cospi_p16_m16, &cospi_p16_p16, &out[2], &out[5]);
unpack_butter_fly(&v4, &v3, &cospi_p16_m16, &cospi_p16_p16, &out[3], &out[4]);
}
static INLINE void add_sub_butterfly(const __m256i *in, __m256i *out,
int size) {
int i = 0;
const int num = size >> 1;
const int bound = size - 1;
while (i < num) {
out[i] = _mm256_add_epi16(in[i], in[bound - i]);
out[bound - i] = _mm256_sub_epi16(in[i], in[bound - i]);
i++;
}
}
static INLINE void idct16_10(__m256i *in /*in[16]*/) {
__m256i out[16];
idct16_10_first_half(in, out);
idct16_10_second_half(in, &out[8]);
add_sub_butterfly(out, in, 16);
}
void aom_idct16x16_10_add_avx2(const tran_low_t *input, uint8_t *dest,
int stride) {
__m256i in[16];
load_coeff(input, &in[0]);
load_coeff(input + 16, &in[1]);
load_coeff(input + 32, &in[2]);
load_coeff(input + 48, &in[3]);
transpose_col_to_row_nz4x4(in);
idct16_10(in);
transpose_col_to_row_nz4x16(in);
idct16_10(in);
write_buffer_16x16(in, stride, dest);
}
// Note:
// For 16x16 int16_t matrix
// transpose first 8 columns into first 8 rows.
// Since only upper-left 8x8 are non-zero, the input are first 8 rows (in[8]).
// After transposing, the 8 row vectors are in in[8].
void transpose_col_to_row_nz8x8(__m256i *in /*in[8]*/) {
__m256i u0 = _mm256_unpacklo_epi16(in[0], in[1]);
__m256i u1 = _mm256_unpackhi_epi16(in[0], in[1]);
__m256i u2 = _mm256_unpacklo_epi16(in[2], in[3]);
__m256i u3 = _mm256_unpackhi_epi16(in[2], in[3]);
const __m256i v0 = _mm256_unpacklo_epi32(u0, u2);
const __m256i v1 = _mm256_unpackhi_epi32(u0, u2);
const __m256i v2 = _mm256_unpacklo_epi32(u1, u3);
const __m256i v3 = _mm256_unpackhi_epi32(u1, u3);
u0 = _mm256_unpacklo_epi16(in[4], in[5]);
u1 = _mm256_unpackhi_epi16(in[4], in[5]);
u2 = _mm256_unpacklo_epi16(in[6], in[7]);
u3 = _mm256_unpackhi_epi16(in[6], in[7]);
const __m256i v4 = _mm256_unpacklo_epi32(u0, u2);
const __m256i v5 = _mm256_unpackhi_epi32(u0, u2);
const __m256i v6 = _mm256_unpacklo_epi32(u1, u3);
const __m256i v7 = _mm256_unpackhi_epi32(u1, u3);
in[0] = SHUFFLE_EPI64(v0, v4, 0);
in[1] = SHUFFLE_EPI64(v0, v4, 3);
in[2] = SHUFFLE_EPI64(v1, v5, 0);
in[3] = SHUFFLE_EPI64(v1, v5, 3);
in[4] = SHUFFLE_EPI64(v2, v6, 0);
in[5] = SHUFFLE_EPI64(v2, v6, 3);
in[6] = SHUFFLE_EPI64(v3, v7, 0);
in[7] = SHUFFLE_EPI64(v3, v7, 3);
}
// Note:
// For 16x16 int16_t matrix
// transpose first 8 columns into first 8 rows.
// Since only matrix left 8x16 are non-zero, the input are total 16 rows
// (in[16]).
// After transposing, the 8 row vectors are in in[8]. All else are zero.
static INLINE void transpose_col_to_row_nz8x16(__m256i *in /*in[16]*/) {
transpose_col_to_row_nz8x8(in);
transpose_col_to_row_nz8x8(&in[8]);
int i;
for (i = 0; i < 8; ++i) {
in[i] = _mm256_permute2x128_si256(in[i], in[i + 8], 0x20);
}
}
static INLINE void idct16_38_first_half(const __m256i *in, __m256i *out) {
const __m256i c2p28 = pair256_set_epi16(2 * cospi_28_64, 2 * cospi_28_64);
const __m256i c2p04 = pair256_set_epi16(2 * cospi_4_64, 2 * cospi_4_64);
__m256i t4 = _mm256_mulhrs_epi16(in[2], c2p28);
__m256i t7 = _mm256_mulhrs_epi16(in[2], c2p04);
const __m256i c2m20 = pair256_set_epi16(-2 * cospi_20_64, -2 * cospi_20_64);
const __m256i c2p12 = pair256_set_epi16(2 * cospi_12_64, 2 * cospi_12_64);
__m256i t5 = _mm256_mulhrs_epi16(in[6], c2m20);
__m256i t6 = _mm256_mulhrs_epi16(in[6], c2p12);
const __m256i c2p16 = pair256_set_epi16(2 * cospi_16_64, 2 * cospi_16_64);
const __m256i c2p24 = pair256_set_epi16(2 * cospi_24_64, 2 * cospi_24_64);
const __m256i c2p08 = pair256_set_epi16(2 * cospi_8_64, 2 * cospi_8_64);
const __m256i u0 = _mm256_mulhrs_epi16(in[0], c2p16);
const __m256i u1 = _mm256_mulhrs_epi16(in[0], c2p16);
const __m256i u2 = _mm256_mulhrs_epi16(in[4], c2p24);
const __m256i u3 = _mm256_mulhrs_epi16(in[4], c2p08);
const __m256i u4 = _mm256_add_epi16(t4, t5);
const __m256i u5 = _mm256_sub_epi16(t4, t5);
const __m256i u6 = _mm256_sub_epi16(t7, t6);
const __m256i u7 = _mm256_add_epi16(t7, t6);
const __m256i t0 = _mm256_add_epi16(u0, u3);
const __m256i t1 = _mm256_add_epi16(u1, u2);
const __m256i t2 = _mm256_sub_epi16(u1, u2);
const __m256i t3 = _mm256_sub_epi16(u0, u3);
t4 = u4;
t7 = u7;
const __m256i cospi_p16_p16 = _mm256_set1_epi16((int16_t)cospi_16_64);
const __m256i cospi_p16_m16 = pair256_set_epi16(cospi_16_64, -cospi_16_64);
unpack_butter_fly(&u6, &u5, &cospi_p16_m16, &cospi_p16_p16, &t5, &t6);
out[0] = _mm256_add_epi16(t0, t7);
out[1] = _mm256_add_epi16(t1, t6);
out[2] = _mm256_add_epi16(t2, t5);
out[3] = _mm256_add_epi16(t3, t4);
out[4] = _mm256_sub_epi16(t3, t4);
out[5] = _mm256_sub_epi16(t2, t5);
out[6] = _mm256_sub_epi16(t1, t6);
out[7] = _mm256_sub_epi16(t0, t7);
}
static INLINE void idct16_38_second_half(const __m256i *in, __m256i *out) {
const __m256i c2p30 = pair256_set_epi16(2 * cospi_30_64, 2 * cospi_30_64);
const __m256i c2p02 = pair256_set_epi16(2 * cospi_2_64, 2 * cospi_2_64);
__m256i t0 = _mm256_mulhrs_epi16(in[1], c2p30);
__m256i t7 = _mm256_mulhrs_epi16(in[1], c2p02);
const __m256i c2m18 = pair256_set_epi16(-2 * cospi_18_64, -2 * cospi_18_64);
const __m256i c2p14 = pair256_set_epi16(2 * cospi_14_64, 2 * cospi_14_64);
__m256i t1 = _mm256_mulhrs_epi16(in[7], c2m18);
__m256i t6 = _mm256_mulhrs_epi16(in[7], c2p14);
const __m256i c2p22 = pair256_set_epi16(2 * cospi_22_64, 2 * cospi_22_64);
const __m256i c2p10 = pair256_set_epi16(2 * cospi_10_64, 2 * cospi_10_64);
__m256i t2 = _mm256_mulhrs_epi16(in[5], c2p22);
__m256i t5 = _mm256_mulhrs_epi16(in[5], c2p10);
const __m256i c2m26 = pair256_set_epi16(-2 * cospi_26_64, -2 * cospi_26_64);
const __m256i c2p06 = pair256_set_epi16(2 * cospi_6_64, 2 * cospi_6_64);
__m256i t3 = _mm256_mulhrs_epi16(in[3], c2m26);
__m256i t4 = _mm256_mulhrs_epi16(in[3], c2p06);
__m256i v0, v1, v2, v3, v4, v5, v6, v7;
v0 = _mm256_add_epi16(t0, t1);
v1 = _mm256_sub_epi16(t0, t1);
v2 = _mm256_sub_epi16(t3, t2);
v3 = _mm256_add_epi16(t2, t3);
v4 = _mm256_add_epi16(t4, t5);
v5 = _mm256_sub_epi16(t4, t5);
v6 = _mm256_sub_epi16(t7, t6);
v7 = _mm256_add_epi16(t6, t7);
t0 = v0;
t7 = v7;
t3 = v3;
t4 = v4;
const __m256i cospi_m08_p24 = pair256_set_epi16(-cospi_8_64, cospi_24_64);
const __m256i cospi_p24_p08 = pair256_set_epi16(cospi_24_64, cospi_8_64);
const __m256i cospi_m24_m08 = pair256_set_epi16(-cospi_24_64, -cospi_8_64);
unpack_butter_fly(&v1, &v6, &cospi_m08_p24, &cospi_p24_p08, &t1, &t6);
unpack_butter_fly(&v2, &v5, &cospi_m24_m08, &cospi_m08_p24, &t2, &t5);
v0 = _mm256_add_epi16(t0, t3);
v1 = _mm256_add_epi16(t1, t2);
v2 = _mm256_sub_epi16(t1, t2);
v3 = _mm256_sub_epi16(t0, t3);
v4 = _mm256_sub_epi16(t7, t4);
v5 = _mm256_sub_epi16(t6, t5);
v6 = _mm256_add_epi16(t6, t5);
v7 = _mm256_add_epi16(t7, t4);
// stage 6, (8-15)
out[0] = v0;
out[1] = v1;
out[6] = v6;
out[7] = v7;
const __m256i cospi_p16_p16 = _mm256_set1_epi16((int16_t)cospi_16_64);
const __m256i cospi_p16_m16 = pair256_set_epi16(cospi_16_64, -cospi_16_64);
unpack_butter_fly(&v5, &v2, &cospi_p16_m16, &cospi_p16_p16, &out[2], &out[5]);
unpack_butter_fly(&v4, &v3, &cospi_p16_m16, &cospi_p16_p16, &out[3], &out[4]);
}
static INLINE void idct16_38(__m256i *in /*in[16]*/) {
__m256i out[16];
idct16_38_first_half(in, out);
idct16_38_second_half(in, &out[8]);
add_sub_butterfly(out, in, 16);
}
void aom_idct16x16_38_add_avx2(const tran_low_t *input, uint8_t *dest,
int stride) {
__m256i in[16];
int i;
for (i = 0; i < 8; ++i) {
load_coeff(input + (i << 4), &in[i]);
}
transpose_col_to_row_nz8x8(in);
idct16_38(in);
transpose_col_to_row_nz8x16(in);
idct16_38(in);
write_buffer_16x16(in, stride, dest);
}
void aom_idct16x16_1_add_avx2(const tran_low_t *input, uint8_t *dest,
int stride) {
__m256i dc_value;
int a, i;
a = (int)dct_const_round_shift(input[0] * cospi_16_64);
a = (int)dct_const_round_shift(a * cospi_16_64);
a = ROUND_POWER_OF_TWO(a, IDCT_ROUNDING_POS);
if (a == 0) return;
dc_value = _mm256_set1_epi16(a);
for (i = 0; i < 16; ++i) {
recon_and_store(&dc_value, dest);
dest += stride;
}
}
/*
* Copyright (c) 2017, Alliance for Open Media. All rights reserved
*
* This source code is subject to the terms of the BSD 2 Clause License and
* the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
* was not distributed with this source code in the LICENSE file, you can
* obtain it at www.aomedia.org/license/software. If the Alliance for Open
* Media Patent License 1.0 was not distributed with this source code in the
* PATENTS file, you can obtain it at www.aomedia.org/license/patent.
*/
#ifndef AOM_DSP_X86_INV_TXFM_COMMON_AVX2_H
#define AOM_DSP_X86_INV_TXFM_COMMON_AVX2_H
#include <immintrin.h>
#include "aom_dsp/txfm_common.h"
#include "aom_dsp/x86/txfm_common_avx2.h"
static INLINE void load_coeff(const tran_low_t *coeff, __m256i *in) {
#if CONFIG_HIGHBITDEPTH
*in = _mm256_setr_epi16(
(int16_t)coeff[0], (int16_t)coeff[1], (int16_t)coeff[2],
(int16_t)coeff[3], (int16_t)coeff[4], (int16_t)coeff[5],
(int16_t)coeff[6], (int16_t)coeff[7], (int16_t)coeff[8],
(int16_t)coeff[9], (int16_t)coeff[10], (int16_t)coeff[11],
(int16_t)coeff[12], (int16_t)coeff[13], (int16_t)coeff[14],
(int16_t)coeff[15]);
#else
*in = _mm256_loadu_si256((const __m256i *)coeff);
#endif
}
static INLINE void load_buffer_16x16(const tran_low_t *coeff, __m256i *in) {
int i = 0;
while (i < 16) {
load_coeff(coeff + (i << 4), &in[i]);
i += 1;
}
}
static INLINE void recon_and_store(const __m256i *res, uint8_t *output) {
const __m128i zero = _mm_setzero_si128();
__m128i x = _mm_loadu_si128((__m128i const *)output);
__m128i p0 = _mm_unpacklo_epi8(x, zero);
__m128i p1 = _mm_unpackhi_epi8(x, zero);
p0 = _mm_add_epi16(p0, _mm256_castsi256_si128(*res));
p1 = _mm_add_epi16(p1, _mm256_extractf128_si256(*res, 1));
x = _mm_packus_epi16(p0, p1);
_mm_storeu_si128((__m128i *)output, x);
}
#define IDCT_ROUNDING_POS (6)
static INLINE void write_buffer_16x16(__m256i *in, const int stride,
uint8_t *output) {
const __m256i rounding = _mm256_set1_epi16(1 << (IDCT_ROUNDING_POS - 1));
int i = 0;
while (i < 16) {
in[i] = _mm256_add_epi16(in[i], rounding);
in[i] = _mm256_srai_epi16(in[i], IDCT_ROUNDING_POS);
recon_and_store(&in[i], output + i * stride);
i += 1;
}
}
static INLINE void unpack_butter_fly(const __m256i *a0, const __m256i *a1,
const __m256i *c0, const __m256i *c1,
__m256i *b0, __m256i *b1) {
__m256i x0, x1;
x0 = _mm256_unpacklo_epi16(*a0, *a1);
x1 = _mm256_unpackhi_epi16(*a0, *a1);
*b0 = butter_fly(x0, x1, *c0);
*b1 = butter_fly(x0, x1, *c1);
}
void av1_idct16_avx2(__m256i *in);
#endif // AOM_DSP_X86_INV_TXFM_COMMON_AVX2_H
...@@ -1255,6 +1255,8 @@ static void idct16x16_add(const tran_low_t *input, uint8_t *dest, int stride, ...@@ -1255,6 +1255,8 @@ static void idct16x16_add(const tran_low_t *input, uint8_t *dest, int stride,
#if !CONFIG_ADAPT_SCAN #if !CONFIG_ADAPT_SCAN
else if (eob <= 10) else if (eob <= 10)
aom_idct16x16_10_add(input, dest, stride); aom_idct16x16_10_add(input, dest, stride);
else if (eob <= 38)
aom_idct16x16_38_add(input, dest, stride);
#endif #endif
else else
aom_idct16x16_256_add(input, dest, stride); aom_idct16x16_256_add(input, dest, stride);
......
...@@ -14,67 +14,9 @@ ...@@ -14,67 +14,9 @@
#include "./aom_config.h" #include "./aom_config.h"
#include "./av1_rtcd.h" #include "./av1_rtcd.h"
#include "aom_dsp/x86/txfm_common_avx2.h" #include "aom_dsp/x86/inv_txfm_common_avx2.h"
static INLINE void load_coeff(const tran_low_t *coeff, __m256i *in) {
#if CONFIG_HIGHBITDEPTH
*in = _mm256_setr_epi16(
(int16_t)coeff[0], (int16_t)coeff[1], (int16_t)coeff[2],
(int16_t)coeff[3], (int16_t)coeff[4], (int16_t)coeff[5],
(int16_t)coeff[6], (int16_t)coeff[7], (int16_t)coeff[8],
(int16_t)coeff[9], (int16_t)coeff[10], (int16_t)coeff[11],
(int16_t)coeff[12], (int16_t)coeff[13], (int16_t)coeff[14],
(int16_t)coeff[15]);
#else
*in = _mm256_loadu_si256((const __m256i *)coeff);
#endif
}
static void load_buffer_16x16(const tran_low_t *coeff, __m256i *in) {
int i = 0;
while (i < 16) {
load_coeff(coeff + (i << 4), &in[i]);
i += 1;
}
}
static void recon_and_store(const __m256i *res, uint8_t *output) {
const __m128i zero = _mm_setzero_si128();
__m128i x = _mm_loadu_si128((__m128i const *)output);