Commit 1c2be1fa authored by Linfeng Zhang's avatar Linfeng Zhang
Browse files

Implement av1_lowbd_fwd_txfm2d_4x4_sse2

Change-Id: I297b1572e89d5668529855f2436b47af95ea2f80
parent 937dcfbd
......@@ -64,6 +64,13 @@ static INLINE __m128i load_32bit_to_16bit_w4(const int32_t *a) {
return _mm_packs_epi32(a_low, a_low);
}
// Store 4 16 bit values. Sign extend the values.
static INLINE void store_16bit_to_32bit_w4(const __m128i a, int32_t *const b) {
const __m128i a_lo = _mm_unpacklo_epi16(a, a);
const __m128i a_1 = _mm_srai_epi32(a_lo, 16);
_mm_store_si128((__m128i *)b, a_1);
}
// Store 8 16 bit values. Sign extend the values.
static INLINE void store_16bit_to_32bit(__m128i a, int32_t *b) {
const __m128i a_lo = _mm_unpacklo_epi16(a, a);
......@@ -74,20 +81,39 @@ static INLINE void store_16bit_to_32bit(__m128i a, int32_t *b) {
_mm_store_si128((__m128i *)(b + 4), a_2);
}
static INLINE void store_rect_16bit_to_32bit(__m128i a, int32_t *b) {
const __m128i sqrt2_coef = _mm_set1_epi16(NewSqrt2);
const __m128i rounding = _mm_set1_epi32(1 << (NewSqrt2Bits - 1));
__m128i a_lo, a_hi;
a_lo = _mm_unpacklo_epi16(a, _mm_setzero_si128());
a_hi = _mm_unpackhi_epi16(a, _mm_setzero_si128());
a_lo = _mm_madd_epi16(a_lo, sqrt2_coef);
a_hi = _mm_madd_epi16(a_hi, sqrt2_coef);
a_lo = _mm_add_epi32(a_lo, rounding);
a_hi = _mm_add_epi32(a_hi, rounding);
a_lo = _mm_srai_epi32(a_lo, NewSqrt2Bits);
a_hi = _mm_srai_epi32(a_hi, NewSqrt2Bits);
_mm_store_si128((__m128i *)b, a_lo);
_mm_store_si128((__m128i *)(b + 4), a_hi);
static INLINE __m128i scale_round_sse2(const __m128i a, const int scale) {
const __m128i scale_rounding = pair_set_epi16(scale, 1 << (NewSqrt2Bits - 1));
const __m128i b = _mm_madd_epi16(a, scale_rounding);
return _mm_srai_epi32(b, NewSqrt2Bits);
}
static INLINE void store_rect_16bit_to_32bit(const __m128i a,
int32_t *const b) {
const __m128i one = _mm_set1_epi16(1);
const __m128i a_lo = _mm_unpacklo_epi16(a, one);
const __m128i a_hi = _mm_unpackhi_epi16(a, one);
const __m128i b_lo = scale_round_sse2(a_lo, NewSqrt2);
const __m128i b_hi = scale_round_sse2(a_hi, NewSqrt2);
_mm_store_si128((__m128i *)b, b_lo);
_mm_store_si128((__m128i *)(b + 4), b_hi);
}
static INLINE void load_buffer_16bit_to_16bit_w4(const int16_t *const in,
const int stride,
__m128i *const out,
const int out_size) {
for (int i = 0; i < out_size; ++i) {
out[i] = _mm_loadl_epi64((const __m128i *)(in + i * stride));
}
}
static INLINE void load_buffer_16bit_to_16bit_w4_flip(const int16_t *const in,
const int stride,
__m128i *const out,
const int out_size) {
for (int i = 0; i < out_size; ++i) {
out[out_size - i - 1] = _mm_loadl_epi64((const __m128i *)(in + i * stride));
}
}
static INLINE void load_buffer_16bit_to_16bit(const int16_t *in, int stride,
......@@ -127,6 +153,15 @@ static INLINE void load_buffer_32bit_to_16bit_flip(const int32_t *in,
}
}
static INLINE void store_buffer_16bit_to_32bit_w4(const __m128i *const in,
int32_t *const out,
const int stride,
const int out_size) {
for (int i = 0; i < out_size; ++i) {
store_16bit_to_32bit_w4(in[i], out + i * stride);
}
}
static INLINE void store_buffer_16bit_to_32bit_8x8(const __m128i *const in,
int32_t *const out,
const int stride) {
......@@ -172,6 +207,9 @@ static INLINE void flip_buf_sse2(__m128i *in, __m128i *out, int size) {
}
}
void av1_lowbd_fwd_txfm2d_4x4_sse2(const int16_t *input, int32_t *output,
int stride, TX_TYPE tx_type, int bd);
void av1_lowbd_fwd_txfm2d_8x8_sse2(const int16_t *input, int32_t *output,
int stride, TX_TYPE tx_type, int bd);
......
......@@ -1303,6 +1303,57 @@ void fdct64_new_sse2(const __m128i *input, __m128i *output, int8_t cos_bit) {
output[63] = x10[63];
}
static void fadst4_new_sse2(const __m128i *input, __m128i *output,
int8_t cos_bit) {
const int32_t *sinpi = sinpi_arr(cos_bit);
const __m128i sinpi_p01_p02 = pair_set_epi16(sinpi[1], sinpi[2]);
const __m128i sinpi_p04_m01 = pair_set_epi16(sinpi[4], -sinpi[1]);
const __m128i sinpi_p03_p04 = pair_set_epi16(sinpi[3], sinpi[4]);
const __m128i sinpi_m03_p02 = pair_set_epi16(-sinpi[3], sinpi[2]);
const __m128i sinpi_p03_p03 = _mm_set1_epi16((int16_t)sinpi[3]);
const __m128i __zero = _mm_set1_epi16(0);
const __m128i __rounding = _mm_set1_epi32(1 << (cos_bit - 1));
const __m128i in7 = _mm_add_epi16(input[0], input[1]);
__m128i u[8], v[8];
u[0] = _mm_unpacklo_epi16(input[0], input[1]);
u[1] = _mm_unpacklo_epi16(input[2], input[3]);
u[2] = _mm_unpacklo_epi16(in7, __zero);
u[3] = _mm_unpacklo_epi16(input[2], __zero);
u[4] = _mm_unpacklo_epi16(input[3], __zero);
v[0] = _mm_madd_epi16(u[0], sinpi_p01_p02); // s0 + s2
v[1] = _mm_madd_epi16(u[1], sinpi_p03_p04); // s4 + s5
v[2] = _mm_madd_epi16(u[2], sinpi_p03_p03); // x1
v[3] = _mm_madd_epi16(u[0], sinpi_p04_m01); // s1 - s3
v[4] = _mm_madd_epi16(u[1], sinpi_m03_p02); // -s4 + s6
v[5] = _mm_madd_epi16(u[3], sinpi_p03_p03); // s4
v[6] = _mm_madd_epi16(u[4], sinpi_p03_p03);
u[0] = _mm_add_epi32(v[0], v[1]);
u[1] = _mm_sub_epi32(v[2], v[6]);
u[2] = _mm_add_epi32(v[3], v[4]);
u[3] = _mm_sub_epi32(u[2], u[0]);
u[4] = _mm_slli_epi32(v[5], 2);
u[5] = _mm_sub_epi32(u[4], v[5]);
u[6] = _mm_add_epi32(u[3], u[5]);
v[0] = _mm_add_epi32(u[0], __rounding);
v[1] = _mm_add_epi32(u[1], __rounding);
v[2] = _mm_add_epi32(u[2], __rounding);
v[3] = _mm_add_epi32(u[6], __rounding);
u[0] = _mm_srai_epi32(v[0], cos_bit);
u[1] = _mm_srai_epi32(v[1], cos_bit);
u[2] = _mm_srai_epi32(v[2], cos_bit);
u[3] = _mm_srai_epi32(v[3], cos_bit);
output[0] = _mm_packs_epi32(u[0], u[2]);
output[1] = _mm_packs_epi32(u[1], u[3]);
output[2] = _mm_srli_si128(output[0], 8);
output[3] = _mm_srli_si128(output[1], 8);
}
void fadst8_new_sse2(const __m128i *input, __m128i *output, int8_t cos_bit) {
const int32_t *cospi = cospi_arr(cos_bit);
const __m128i __zero = _mm_setzero_si128();
......@@ -1575,6 +1626,19 @@ void fadst16_new_sse2(const __m128i *input, __m128i *output, int8_t cos_bit) {
output[15] = x8[0];
}
static INLINE void fidentity4_new_sse2(const __m128i *const input,
__m128i *const output,
const int8_t cos_bit) {
(void)cos_bit;
const __m128i one = _mm_set1_epi16(1);
for (int i = 0; i < 4; ++i) {
const __m128i a = _mm_unpacklo_epi16(input[i], one);
const __m128i b = scale_round_sse2(a, NewSqrt2);
output[i] = _mm_packs_epi32(b, b);
}
}
static INLINE void fidentity8_new_sse2(const __m128i *input, __m128i *output,
int8_t cos_bit) {
(void)cos_bit;
......@@ -1592,18 +1656,14 @@ static INLINE void fidentity8_new_sse2(const __m128i *input, __m128i *output,
static INLINE void fidentity16_new_sse2(const __m128i *input, __m128i *output,
int8_t cos_bit) {
(void)cos_bit;
const __m128i scale = _mm_set1_epi16(2 * NewSqrt2);
const __m128i rounding = _mm_set1_epi16(1 << (NewSqrt2Bits - 1));
const __m128i one = _mm_set1_epi16(1);
const __m128i scale_rounding = _mm_unpacklo_epi16(scale, rounding);
for (int i = 0; i < 16; ++i) {
__m128i a_lo = _mm_unpacklo_epi16(input[i], one);
__m128i a_hi = _mm_unpackhi_epi16(input[i], one);
__m128i b_lo = _mm_madd_epi16(a_lo, scale_rounding);
__m128i b_hi = _mm_madd_epi16(a_hi, scale_rounding);
__m128i c_lo = _mm_srai_epi32(b_lo, NewSqrt2Bits);
__m128i c_hi = _mm_srai_epi32(b_hi, NewSqrt2Bits);
output[i] = _mm_packs_epi32(c_lo, c_hi);
const __m128i a_lo = _mm_unpacklo_epi16(input[i], one);
const __m128i a_hi = _mm_unpackhi_epi16(input[i], one);
const __m128i b_lo = scale_round_sse2(a_lo, 2 * NewSqrt2);
const __m128i b_hi = scale_round_sse2(a_hi, 2 * NewSqrt2);
output[i] = _mm_packs_epi32(b_lo, b_hi);
}
}
......@@ -1615,6 +1675,25 @@ static INLINE void fidentity32_new_sse2(const __m128i *input, __m128i *output,
}
}
static const transform_2d_sse2 txfm4_arr[] = {
{ fdct4_new_sse2, fdct4_new_sse2 }, // DCT_DCT
{ fadst4_new_sse2, fdct4_new_sse2 }, // ADST_DCT
{ fdct4_new_sse2, fadst4_new_sse2 }, // DCT_ADST
{ fadst4_new_sse2, fadst4_new_sse2 }, // ADST_ADST
{ fadst4_new_sse2, fdct4_new_sse2 }, // FLIPADST_DCT
{ fdct4_new_sse2, fadst4_new_sse2 }, // DCT_FLIPADST
{ fadst4_new_sse2, fadst4_new_sse2 }, // FLIPADST_FLIPADST
{ fadst4_new_sse2, fadst4_new_sse2 }, // ADST_FLIPADST
{ fadst4_new_sse2, fadst4_new_sse2 }, // FLIPADST_ADST
{ fidentity4_new_sse2, fidentity4_new_sse2 }, // IDTX
{ fdct4_new_sse2, fidentity4_new_sse2 }, // V_DCT
{ fidentity4_new_sse2, fdct4_new_sse2 }, // H_DCT
{ fadst4_new_sse2, fidentity4_new_sse2 }, // V_ADST
{ fidentity4_new_sse2, fadst4_new_sse2 }, // H_ADST
{ fadst4_new_sse2, fidentity4_new_sse2 }, // V_FLIPADST
{ fidentity4_new_sse2, fadst4_new_sse2 }, // H_FLIPADST
};
static const transform_2d_sse2 txfm8_arr[] = {
{ fdct8_new_sse2, fdct8_new_sse2 }, // DCT_DCT
{ fadst8_new_sse2, fdct8_new_sse2 }, // ADST_DCT
......@@ -1672,40 +1751,79 @@ static const transform_2d_sse2 txfm32_arr[] = {
{ NULL, NULL }, // H_FLIPADST
};
void av1_lowbd_fwd_txfm2d_4x4_sse2(const int16_t *input, int32_t *output,
int stride, TX_TYPE tx_type, int bd) {
(void)bd;
__m128i buf0[4], buf1[4], *buf;
const int8_t *shift = fwd_txfm_shift_ls[TX_4X4];
const int txw_idx = get_txw_idx(TX_4X4);
const int txh_idx = get_txh_idx(TX_4X4);
const int cos_bit_col = fwd_cos_bit_col[txw_idx][txh_idx];
const int cos_bit_row = fwd_cos_bit_row[txw_idx][txh_idx];
const int width = 4;
const int height = 4;
const transform_1d_sse2 col_txfm = txfm4_arr[tx_type].col;
const transform_1d_sse2 row_txfm = txfm4_arr[tx_type].row;
int ud_flip, lr_flip;
get_flip_cfg(tx_type, &ud_flip, &lr_flip);
if (ud_flip) {
load_buffer_16bit_to_16bit_w4_flip(input, stride, buf0, height);
} else {
load_buffer_16bit_to_16bit_w4(input, stride, buf0, height);
}
round_shift_16bit(buf0, height, shift[0]);
col_txfm(buf0, buf0, cos_bit_col);
round_shift_16bit(buf0, height, shift[1]);
transpose_16bit_4x4(buf0, buf1);
if (lr_flip) {
buf = buf0;
flip_buf_sse2(buf1, buf, width);
} else {
buf = buf1;
}
row_txfm(buf, buf, cos_bit_row);
round_shift_16bit(buf, width, shift[2]);
transpose_16bit_4x4(buf, buf);
store_buffer_16bit_to_32bit_w4(buf, output, width, height);
}
void av1_lowbd_fwd_txfm2d_8x8_sse2(const int16_t *input, int32_t *output,
int stride, TX_TYPE tx_type, int bd) {
(void)stride;
(void)bd;
__m128i buf[8];
__m128i buf0[8], buf1[8], *buf;
const int8_t *shift = fwd_txfm_shift_ls[TX_8X8];
const int txw_idx = get_txw_idx(TX_8X8);
const int txh_idx = get_txh_idx(TX_8X8);
const int cos_bit_col = fwd_cos_bit_col[txw_idx][txh_idx];
const int cos_bit_row = fwd_cos_bit_row[txw_idx][txh_idx];
const int buf_size = 8;
const int width = 8;
const int height = 8;
const transform_1d_sse2 col_txfm = txfm8_arr[tx_type].col;
const transform_1d_sse2 row_txfm = txfm8_arr[tx_type].row;
int ud_flip, lr_flip;
get_flip_cfg(tx_type, &ud_flip, &lr_flip);
if (ud_flip)
load_buffer_16bit_to_16bit_flip(input, stride, buf, buf_size);
load_buffer_16bit_to_16bit_flip(input, stride, buf0, height);
else
load_buffer_16bit_to_16bit(input, stride, buf, buf_size);
round_shift_16bit(buf, 8, shift[0]);
col_txfm(buf, buf, cos_bit_col);
round_shift_16bit(buf, 8, shift[1]);
load_buffer_16bit_to_16bit(input, stride, buf0, height);
round_shift_16bit(buf0, height, shift[0]);
col_txfm(buf0, buf0, cos_bit_col);
round_shift_16bit(buf0, height, shift[1]);
transpose_16bit_8x8(buf0, buf1);
if (lr_flip) {
__m128i tmp[8];
transpose_16bit_8x8(buf, tmp);
flip_buf_sse2(tmp, buf, 8);
buf = buf0;
flip_buf_sse2(buf1, buf, width);
} else {
transpose_16bit_8x8(buf, buf);
buf = buf1;
}
row_txfm(buf, buf, cos_bit_row);
round_shift_16bit(buf, 8, shift[2]);
round_shift_16bit(buf, width, shift[2]);
transpose_16bit_8x8(buf, buf);
store_buffer_16bit_to_32bit_8x8(buf, output, buf_size);
store_buffer_16bit_to_32bit_8x8(buf, output, width);
}
void av1_lowbd_fwd_txfm2d_8x16_sse2(const int16_t *input, int32_t *output,
......@@ -1721,10 +1839,9 @@ void av1_lowbd_fwd_txfm2d_8x16_sse2(const int16_t *input, int32_t *output,
const int height = 16;
const transform_1d_sse2 col_txfm = txfm16_arr[tx_type].col;
const transform_1d_sse2 row_txfm = txfm8_arr[tx_type].row;
int ud_flip, lr_flip;
get_flip_cfg(tx_type, &ud_flip, &lr_flip);
get_flip_cfg(tx_type, &ud_flip, &lr_flip);
if (ud_flip) {
load_buffer_16bit_to_16bit_flip(input, stride, buf0, height);
} else {
......@@ -1764,10 +1881,9 @@ void av1_lowbd_fwd_txfm2d_8x32_sse2(const int16_t *input, int32_t *output,
const int height = 32;
const transform_1d_sse2 col_txfm = txfm32_arr[tx_type].col;
const transform_1d_sse2 row_txfm = txfm8_arr[tx_type].row;
int ud_flip, lr_flip;
get_flip_cfg(tx_type, &ud_flip, &lr_flip);
get_flip_cfg(tx_type, &ud_flip, &lr_flip);
if (ud_flip) {
load_buffer_16bit_to_16bit_flip(input, stride, buf0, height);
} else {
......@@ -1809,11 +1925,10 @@ void av1_lowbd_fwd_txfm2d_16x8_sse2(const int16_t *input, int32_t *output,
const int height = 8;
const transform_1d_sse2 col_txfm = txfm8_arr[tx_type].col;
const transform_1d_sse2 row_txfm = txfm16_arr[tx_type].row;
__m128i *buf;
int ud_flip, lr_flip;
get_flip_cfg(tx_type, &ud_flip, &lr_flip);
get_flip_cfg(tx_type, &ud_flip, &lr_flip);
for (int i = 0; i < 2; i++) {
if (ud_flip) {
load_buffer_16bit_to_16bit_flip(input + 8 * i, stride, buf0, height);
......@@ -1853,10 +1968,9 @@ void av1_lowbd_fwd_txfm2d_16x16_sse2(const int16_t *input, int32_t *output,
const int height = 16;
const transform_1d_sse2 col_txfm = txfm16_arr[tx_type].col;
const transform_1d_sse2 row_txfm = txfm16_arr[tx_type].row;
int ud_flip, lr_flip;
get_flip_cfg(tx_type, &ud_flip, &lr_flip);
get_flip_cfg(tx_type, &ud_flip, &lr_flip);
for (int i = 0; i < 2; i++) {
if (ud_flip) {
load_buffer_16bit_to_16bit_flip(input + 8 * i, stride, buf0, height);
......@@ -2120,7 +2234,7 @@ typedef void (*FwdTxfm2dFuncSSE2)(const int16_t *input, int32_t *output,
int stride, TX_TYPE tx_type, int bd);
FwdTxfm2dFuncSSE2 fwd_txfm2d_func_ls[TX_SIZES_ALL] = {
NULL, // 4x4 transform
av1_lowbd_fwd_txfm2d_4x4_sse2, // 4x4 transform
av1_lowbd_fwd_txfm2d_8x8_sse2, // 8x8 transform
av1_lowbd_fwd_txfm2d_16x16_sse2, // 16x16 transform
av1_lowbd_fwd_txfm2d_32x32_sse2, // 32x32 transform
......@@ -2151,9 +2265,10 @@ void av1_lowbd_fwd_txfm_sse2(const int16_t *src_diff, tran_low_t *coeff,
int diff_stride, TxfmParam *txfm_param) {
FwdTxfm2dFuncSSE2 fwd_txfm2d_func = fwd_txfm2d_func_ls[txfm_param->tx_size];
if (fwd_txfm2d_func)
if ((fwd_txfm2d_func == NULL) ||
(txfm_param->lossless && txfm_param->tx_size == TX_4X4))
av1_lowbd_fwd_txfm_c(src_diff, coeff, diff_stride, txfm_param);
else
fwd_txfm2d_func(src_diff, coeff, diff_stride, txfm_param->tx_type,
txfm_param->bd);
else
av1_lowbd_fwd_txfm_c(src_diff, coeff, diff_stride, txfm_param);
}
......@@ -211,7 +211,7 @@ TEST(AV1FwdTxfm2d, CfgTest) {
#if HAVE_SSE2 && defined(__SSE2__)
#include "av1/common/x86/av1_txfm_sse2.h"
FwdTxfm2dFunc fwd_func_sse2_list[TX_SIZES_ALL][2] = {
{ NULL, NULL }, // TX_4X4
{ av1_fwd_txfm2d_4x4_c, av1_lowbd_fwd_txfm2d_4x4_sse2 }, // TX_4X4
{ av1_fwd_txfm2d_8x8_c, av1_lowbd_fwd_txfm2d_8x8_sse2 }, // TX_8X8
{ av1_fwd_txfm2d_16x16_c, av1_lowbd_fwd_txfm2d_16x16_sse2 }, // TX_16X16
{ av1_fwd_txfm2d_32x32_c, av1_lowbd_fwd_txfm2d_32x32_sse2 }, // TX_32X32
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment