Commit ea8a0d52 authored by Yi Luo's avatar Yi Luo

Lowbd parallel_deblocking sse2 optimization

Baseline + parallel_deblocking:

- Passed unit tests *SSE2/Loop8Test6*, *AVX2/Loop8Test6*.
- 1080p, 25 frames, profile=0, encoding/decoding, output match.
- Decoder frame rate increases from 54.15 to 65.84.

Change-Id: I55938c94961066594f4b9080192c7268c19d9bf9
parent 07ed3ab2
......@@ -65,12 +65,17 @@ set(AOM_DSP_COMMON_INTRIN_SSE4_1
set(AOM_DSP_COMMON_INTRIN_AVX2
"${AOM_ROOT}/aom_dsp/x86/aom_subpixel_8t_intrin_avx2.c"
"${AOM_ROOT}/aom_dsp/x86/loopfilter_avx2.c"
"${AOM_ROOT}/aom_dsp/x86/inv_txfm_avx2.c"
"${AOM_ROOT}/aom_dsp/x86/common_avx2.h"
"${AOM_ROOT}/aom_dsp/x86/inv_txfm_common_avx2.h"
"${AOM_ROOT}/aom_dsp/x86/txfm_common_avx2.h")
if (NOT CONFIG_PARALLEL_DEBLOCKING)
set(AOM_DSP_COMMON_INTRIN_AVX2
${AOM_DSP_COMMON_INTRIN_AVX2}
"${AOM_ROOT}/aom_dsp/x86/loopfilter_avx2.c")
endif ()
set(AOM_DSP_COMMON_ASM_NEON
"${AOM_ROOT}/aom_dsp/arm/aom_convolve8_avg_neon_asm.asm"
"${AOM_ROOT}/aom_dsp/arm/aom_convolve8_neon_asm.asm"
......@@ -85,12 +90,17 @@ set(AOM_DSP_COMMON_ASM_NEON
"${AOM_ROOT}/aom_dsp/arm/idct8x8_1_add_neon.asm"
"${AOM_ROOT}/aom_dsp/arm/idct8x8_add_neon.asm"
"${AOM_ROOT}/aom_dsp/arm/intrapred_neon_asm.asm"
"${AOM_ROOT}/aom_dsp/arm/loopfilter_16_neon.asm"
"${AOM_ROOT}/aom_dsp/arm/loopfilter_4_neon.asm"
"${AOM_ROOT}/aom_dsp/arm/loopfilter_8_neon.asm"
"${AOM_ROOT}/aom_dsp/arm/loopfilter_mb_neon.asm"
"${AOM_ROOT}/aom_dsp/arm/save_reg_neon.asm")
if (NOT CONFIG_PARALLEL_DEBLOCKING)
set(AOM_DSP_COMMON_ASM_NEON
${AOM_DSP_COMMON_ASM_NEON}
"${AOM_ROOT}/aom_dsp/arm/loopfilter_16_neon.asm"
"${AOM_ROOT}/aom_dsp/arm/loopfilter_4_neon.asm"
"${AOM_ROOT}/aom_dsp/arm/loopfilter_8_neon.asm"
"${AOM_ROOT}/aom_dsp/arm/loopfilter_mb_neon.asm")
endif ()
set(AOM_DSP_COMMON_INTRIN_NEON
"${AOM_ROOT}/aom_dsp/arm/aom_convolve_neon.c"
"${AOM_ROOT}/aom_dsp/arm/avg_neon.c"
......@@ -98,13 +108,18 @@ set(AOM_DSP_COMMON_INTRIN_NEON
"${AOM_ROOT}/aom_dsp/arm/hadamard_neon.c"
"${AOM_ROOT}/aom_dsp/arm/idct16x16_neon.c"
"${AOM_ROOT}/aom_dsp/arm/intrapred_neon.c"
"${AOM_ROOT}/aom_dsp/arm/loopfilter_neon.c"
"${AOM_ROOT}/aom_dsp/arm/sad4d_neon.c"
"${AOM_ROOT}/aom_dsp/arm/sad_neon.c"
"${AOM_ROOT}/aom_dsp/arm/subpel_variance_neon.c"
"${AOM_ROOT}/aom_dsp/arm/subtract_neon.c"
"${AOM_ROOT}/aom_dsp/arm/variance_neon.c")
if (NOT CONFIG_PARALLEL_DEBLOCKING)
set(AOM_DSP_COMMON_INTRIN_NEON
${AOM_DSP_COMMON_INTRIN_NEON}
"${AOM_ROOT}/aom_dsp/arm/loopfilter_neon.c")
endif ()
if ("${AOM_TARGET_CPU}" STREQUAL "arm64")
set(AOM_DSP_COMMON_INTRIN_NEON
${AOM_DSP_COMMON_INTRIN_NEON}
......@@ -120,10 +135,15 @@ if ("${AOM_TARGET_CPU}" STREQUAL "arm64")
"${AOM_ROOT}/aom_dsp/arm/idct4x4_add_neon.c"
"${AOM_ROOT}/aom_dsp/arm/idct8x8_1_add_neon.c"
"${AOM_ROOT}/aom_dsp/arm/idct8x8_add_neon.c"
"${AOM_ROOT}/aom_dsp/arm/intrapred_neon.c"
"${AOM_ROOT}/aom_dsp/arm/loopfilter_16_neon.c"
"${AOM_ROOT}/aom_dsp/arm/loopfilter_4_neon.c"
"${AOM_ROOT}/aom_dsp/arm/loopfilter_8_neon.c")
"${AOM_ROOT}/aom_dsp/arm/intrapred_neon.c")
if (NOT CONFIG_PARALLEL_DEBLOCKING)
set(AOM_DSP_COMMON_INTRIN_NEON
${AOM_DSP_COMMON_INTRIN_NEON}
"${AOM_ROOT}/aom_dsp/arm/loopfilter_16_neon.c"
"${AOM_ROOT}/aom_dsp/arm/loopfilter_4_neon.c"
"${AOM_ROOT}/aom_dsp/arm/loopfilter_8_neon.c")
endif ()
endif ()
set(AOM_DSP_COMMON_INTRIN_DSPR2
......@@ -143,14 +163,19 @@ set(AOM_DSP_COMMON_INTRIN_DSPR2
"${AOM_ROOT}/aom_dsp/mips/intrapred16_dspr2.c"
"${AOM_ROOT}/aom_dsp/mips/intrapred4_dspr2.c"
"${AOM_ROOT}/aom_dsp/mips/intrapred8_dspr2.c"
"${AOM_ROOT}/aom_dsp/mips/inv_txfm_dspr2.h"
"${AOM_ROOT}/aom_dsp/mips/loopfilter_filters_dspr2.c"
"${AOM_ROOT}/aom_dsp/mips/loopfilter_filters_dspr2.h"
"${AOM_ROOT}/aom_dsp/mips/loopfilter_macros_dspr2.h"
"${AOM_ROOT}/aom_dsp/mips/loopfilter_masks_dspr2.h"
"${AOM_ROOT}/aom_dsp/mips/loopfilter_mb_dspr2.c"
"${AOM_ROOT}/aom_dsp/mips/loopfilter_mb_horiz_dspr2.c"
"${AOM_ROOT}/aom_dsp/mips/loopfilter_mb_vert_dspr2.c")
"${AOM_ROOT}/aom_dsp/mips/inv_txfm_dspr2.h")
if (NOT CONFIG_PARALLEL_DEBLOCKING)
set(AOM_DSP_COMMON_INTRIN_DSPR2
${AOM_DSP_COMMON_INTRIN_DSPR2}
"${AOM_ROOT}/aom_dsp/mips/loopfilter_filters_dspr2.c"
"${AOM_ROOT}/aom_dsp/mips/loopfilter_filters_dspr2.h"
"${AOM_ROOT}/aom_dsp/mips/loopfilter_macros_dspr2.h"
"${AOM_ROOT}/aom_dsp/mips/loopfilter_masks_dspr2.h"
"${AOM_ROOT}/aom_dsp/mips/loopfilter_mb_dspr2.c"
"${AOM_ROOT}/aom_dsp/mips/loopfilter_mb_horiz_dspr2.c"
"${AOM_ROOT}/aom_dsp/mips/loopfilter_mb_vert_dspr2.c")
endif ()
set(AOM_DSP_COMMON_INTRIN_MSA
"${AOM_ROOT}/aom_dsp/mips/aom_convolve8_avg_horiz_msa.c"
......@@ -171,13 +196,18 @@ set(AOM_DSP_COMMON_INTRIN_MSA
"${AOM_ROOT}/aom_dsp/mips/idct8x8_msa.c"
"${AOM_ROOT}/aom_dsp/mips/intrapred_msa.c"
"${AOM_ROOT}/aom_dsp/mips/inv_txfm_msa.h"
"${AOM_ROOT}/aom_dsp/mips/loopfilter_16_msa.c"
"${AOM_ROOT}/aom_dsp/mips/loopfilter_4_msa.c"
"${AOM_ROOT}/aom_dsp/mips/loopfilter_8_msa.c"
"${AOM_ROOT}/aom_dsp/mips/loopfilter_msa.h"
"${AOM_ROOT}/aom_dsp/mips/macros_msa.h"
"${AOM_ROOT}/aom_dsp/mips/txfm_macros_msa.h")
if (NOT CONFIG_PARALLEL_DEBLOCKING)
set(AOM_DSP_COMMON_INTRIN_MSA
${AOM_DSP_COMMON_INTRIN_MSA}
"${AOM_ROOT}/aom_dsp/mips/loopfilter_16_msa.c"
"${AOM_ROOT}/aom_dsp/mips/loopfilter_4_msa.c"
"${AOM_ROOT}/aom_dsp/mips/loopfilter_8_msa.c"
"${AOM_ROOT}/aom_dsp/mips/loopfilter_msa.h")
endif ()
if (CONFIG_HIGHBITDEPTH)
set(AOM_DSP_COMMON_ASM_SSE2
${AOM_DSP_COMMON_ASM_SSE2}
......
......@@ -164,9 +164,11 @@ DSP_SRCS-$(HAVE_DSPR2) += mips/convolve8_vert_dspr2.c
DSP_SRCS-yes += loopfilter.c
DSP_SRCS-$(ARCH_X86)$(ARCH_X86_64) += x86/loopfilter_sse2.c
DSP_SRCS-$(HAVE_AVX2) += x86/loopfilter_avx2.c
DSP_SRCS-$(HAVE_SSE2) += x86/lpf_common_sse2.h
ifneq ($(CONFIG_PARALLEL_DEBLOCKING),yes)
DSP_SRCS-$(HAVE_AVX2) += x86/loopfilter_avx2.c
DSP_SRCS-$(HAVE_NEON) += arm/loopfilter_neon.c
ifeq ($(HAVE_NEON_ASM),yes)
DSP_SRCS-yes += arm/loopfilter_mb_neon$(ASM)
......@@ -192,6 +194,7 @@ DSP_SRCS-$(HAVE_DSPR2) += mips/loopfilter_masks_dspr2.h
DSP_SRCS-$(HAVE_DSPR2) += mips/loopfilter_mb_dspr2.c
DSP_SRCS-$(HAVE_DSPR2) += mips/loopfilter_mb_horiz_dspr2.c
DSP_SRCS-$(HAVE_DSPR2) += mips/loopfilter_mb_vert_dspr2.c
endif # !CONFIG_PARALLEL_DEBLOCKING
ifeq ($(CONFIG_HIGHBITDEPTH),yes)
DSP_SRCS-$(HAVE_SSE2) += x86/highbd_loopfilter_sse2.c
......
......@@ -230,46 +230,84 @@ if (aom_config("CONFIG_HIGHBITDEPTH") eq "yes") {
# Loopfilter
#
add_proto qw/void aom_lpf_vertical_16/, "uint8_t *s, int pitch, const uint8_t *blimit, const uint8_t *limit, const uint8_t *thresh";
specialize qw/aom_lpf_vertical_16 sse2 neon_asm dspr2 msa/;
$aom_lpf_vertical_16_neon_asm=aom_lpf_vertical_16_neon;
if (aom_config("CONFIG_PARALLEL_DEBLOCKING") eq "yes") {
specialize qw/aom_lpf_vertical_16 sse2/;
} else {
specialize qw/aom_lpf_vertical_16 sse2 neon_asm dspr2 msa/;
$aom_lpf_vertical_16_neon_asm=aom_lpf_vertical_16_neon;
}
add_proto qw/void aom_lpf_vertical_16_dual/, "uint8_t *s, int pitch, const uint8_t *blimit, const uint8_t *limit, const uint8_t *thresh";
specialize qw/aom_lpf_vertical_16_dual sse2 neon_asm dspr2 msa/;
$aom_lpf_vertical_16_dual_neon_asm=aom_lpf_vertical_16_dual_neon;
if (aom_config("CONFIG_PARALLEL_DEBLOCKING") ne "yes") {
specialize qw/aom_lpf_vertical_16_dual sse2 neon_asm dspr2 msa/;
$aom_lpf_vertical_16_dual_neon_asm=aom_lpf_vertical_16_dual_neon;
}
add_proto qw/void aom_lpf_vertical_8/, "uint8_t *s, int pitch, const uint8_t *blimit, const uint8_t *limit, const uint8_t *thresh";
specialize qw/aom_lpf_vertical_8 sse2 neon dspr2 msa/;
if (aom_config("CONFIG_PARALLEL_DEBLOCKING") eq "yes") {
specialize qw/aom_lpf_vertical_8 sse2/;
} else {
specialize qw/aom_lpf_vertical_8 sse2 neon dspr2 msa/;
}
add_proto qw/void aom_lpf_vertical_8_dual/, "uint8_t *s, int pitch, const uint8_t *blimit0, const uint8_t *limit0, const uint8_t *thresh0, const uint8_t *blimit1, const uint8_t *limit1, const uint8_t *thresh1";
specialize qw/aom_lpf_vertical_8_dual sse2 neon_asm dspr2 msa/;
$aom_lpf_vertical_8_dual_neon_asm=aom_lpf_vertical_8_dual_neon;
if (aom_config("CONFIG_PARALLEL_DEBLOCKING") ne "yes") {
specialize qw/aom_lpf_vertical_8_dual sse2 neon_asm dspr2 msa/;
$aom_lpf_vertical_8_dual_neon_asm=aom_lpf_vertical_8_dual_neon;
}
add_proto qw/void aom_lpf_vertical_4/, "uint8_t *s, int pitch, const uint8_t *blimit, const uint8_t *limit, const uint8_t *thresh";
specialize qw/aom_lpf_vertical_4 sse2 neon dspr2 msa/;
if (aom_config("CONFIG_PARALLEL_DEBLOCKING") eq "yes") {
specialize qw/aom_lpf_vertical_4 sse2/;
} else {
specialize qw/aom_lpf_vertical_4 sse2 neon dspr2 msa/;
}
add_proto qw/void aom_lpf_vertical_4_dual/, "uint8_t *s, int pitch, const uint8_t *blimit0, const uint8_t *limit0, const uint8_t *thresh0, const uint8_t *blimit1, const uint8_t *limit1, const uint8_t *thresh1";
specialize qw/aom_lpf_vertical_4_dual sse2 neon dspr2 msa/;
if (aom_config("CONFIG_PARALLEL_DEBLOCKING") ne "yes") {
specialize qw/aom_lpf_vertical_4_dual sse2 neon dspr2 msa/;
}
add_proto qw/void aom_lpf_horizontal_edge_8/, "uint8_t *s, int pitch, const uint8_t *blimit, const uint8_t *limit, const uint8_t *thresh";
specialize qw/aom_lpf_horizontal_edge_8 sse2 avx2 neon_asm dspr2 msa/;
$aom_lpf_horizontal_edge_8_neon_asm=aom_lpf_horizontal_edge_8_neon;
if (aom_config("CONFIG_PARALLEL_DEBLOCKING") eq "yes") {
specialize qw/aom_lpf_horizontal_edge_8 sse2/;
} else {
specialize qw/aom_lpf_horizontal_edge_8 sse2 avx2 neon_asm dspr2 msa/;
$aom_lpf_horizontal_edge_8_neon_asm=aom_lpf_horizontal_edge_8_neon;
}
add_proto qw/void aom_lpf_horizontal_edge_16/, "uint8_t *s, int pitch, const uint8_t *blimit, const uint8_t *limit, const uint8_t *thresh";
specialize qw/aom_lpf_horizontal_edge_16 sse2 avx2 neon_asm dspr2 msa/;
$aom_lpf_horizontal_edge_16_neon_asm=aom_lpf_horizontal_edge_16_neon;
if (aom_config("CONFIG_PARALLEL_DEBLOCKING") eq "yes") {
specialize qw/aom_lpf_horizontal_edge_16 sse2/;
} else {
specialize qw/aom_lpf_horizontal_edge_16 sse2 avx2 neon_asm dspr2 msa/;
$aom_lpf_horizontal_edge_16_neon_asm=aom_lpf_horizontal_edge_16_neon;
}
add_proto qw/void aom_lpf_horizontal_8/, "uint8_t *s, int pitch, const uint8_t *blimit, const uint8_t *limit, const uint8_t *thresh";
specialize qw/aom_lpf_horizontal_8 sse2 neon dspr2 msa/;
if (aom_config("CONFIG_PARALLEL_DEBLOCKING") eq "yes") {
specialize qw/aom_lpf_horizontal_8 sse2/;
} else {
specialize qw/aom_lpf_horizontal_8 sse2 neon dspr2 msa/;
}
add_proto qw/void aom_lpf_horizontal_8_dual/, "uint8_t *s, int pitch, const uint8_t *blimit0, const uint8_t *limit0, const uint8_t *thresh0, const uint8_t *blimit1, const uint8_t *limit1, const uint8_t *thresh1";
specialize qw/aom_lpf_horizontal_8_dual sse2 neon_asm dspr2 msa/;
$aom_lpf_horizontal_8_dual_neon_asm=aom_lpf_horizontal_8_dual_neon;
if (aom_config("CONFIG_PARALLEL_DEBLOCKING") ne "yes") {
specialize qw/aom_lpf_horizontal_8_dual sse2 neon_asm dspr2 msa/;
$aom_lpf_horizontal_8_dual_neon_asm=aom_lpf_horizontal_8_dual_neon;
}
add_proto qw/void aom_lpf_horizontal_4/, "uint8_t *s, int pitch, const uint8_t *blimit, const uint8_t *limit, const uint8_t *thresh";
specialize qw/aom_lpf_horizontal_4 sse2 neon dspr2 msa/;
if (aom_config("CONFIG_PARALLEL_DEBLOCKING") eq "yes") {
specialize qw/aom_lpf_horizontal_4 sse2/;
} else {
specialize qw/aom_lpf_horizontal_4 sse2 neon dspr2 msa/;
}
add_proto qw/void aom_lpf_horizontal_4_dual/, "uint8_t *s, int pitch, const uint8_t *blimit0, const uint8_t *limit0, const uint8_t *thresh0, const uint8_t *blimit1, const uint8_t *limit1, const uint8_t *thresh1";
specialize qw/aom_lpf_horizontal_4_dual sse2 neon dspr2 msa/;
if (aom_config("CONFIG_PARALLEL_DEBLOCKING") ne "yes") {
specialize qw/aom_lpf_horizontal_4_dual sse2 neon dspr2 msa/;
}
if (aom_config("CONFIG_HIGHBITDEPTH") eq "yes") {
add_proto qw/void aom_highbd_lpf_vertical_16/, "uint16_t *s, int pitch, const uint8_t *blimit, const uint8_t *limit, const uint8_t *thresh, int bd";
......
......@@ -178,10 +178,20 @@ void aom_lpf_horizontal_4_sse2(uint8_t *s, int p /* pitch */,
#endif // !CONFIG_PARALLEL_DEBLOCKING
FILTER4;
#if CONFIG_PARALLEL_DEBLOCKING
*(int32_t *)(s - 1 * p) = _mm_cvtsi128_si32(ps1ps0);
ps1ps0 = _mm_srli_si128(ps1ps0, 8);
*(int32_t *)(s - 2 * p) = _mm_cvtsi128_si32(ps1ps0);
*(int32_t *)(s + 0 * p) = _mm_cvtsi128_si32(qs1qs0);
qs1qs0 = _mm_srli_si128(qs1qs0, 8);
*(int32_t *)(s + 1 * p) = _mm_cvtsi128_si32(qs1qs0);
#else
_mm_storeh_pi((__m64 *)(s - 2 * p), _mm_castsi128_ps(ps1ps0)); // *op1
_mm_storel_epi64((__m128i *)(s - 1 * p), ps1ps0); // *op0
_mm_storel_epi64((__m128i *)(s + 0 * p), qs1qs0); // *oq0
_mm_storeh_pi((__m64 *)(s + 1 * p), _mm_castsi128_ps(qs1qs0)); // *oq1
#endif
}
void aom_lpf_vertical_4_sse2(uint8_t *s, int p /* pitch */,
......@@ -267,8 +277,10 @@ void aom_lpf_vertical_4_sse2(uint8_t *s, int p /* pitch */,
x0 = _mm_unpackhi_epi8(ps1ps0, qs1qs0);
// 00 20 01 21 02 22 03 23 04 24 05 25 06 26 07 27
ps1ps0 = _mm_unpacklo_epi8(ps1ps0, qs1qs0);
#if !CONFIG_PARALLEL_DEBLOCKING
// 04 14 24 34 05 15 25 35 06 16 26 36 07 17 27 37
qs1qs0 = _mm_unpackhi_epi8(ps1ps0, x0);
#endif
// 00 10 20 30 01 11 21 31 02 12 22 32 03 13 23 33
ps1ps0 = _mm_unpacklo_epi8(ps1ps0, x0);
......@@ -279,7 +291,7 @@ void aom_lpf_vertical_4_sse2(uint8_t *s, int p /* pitch */,
*(int *)(s + 2 * p - 2) = _mm_cvtsi128_si32(ps1ps0);
ps1ps0 = _mm_srli_si128(ps1ps0, 4);
*(int *)(s + 3 * p - 2) = _mm_cvtsi128_si32(ps1ps0);
#if !CONFIG_PARALLEL_DEBLOCKING
*(int *)(s + 4 * p - 2) = _mm_cvtsi128_si32(qs1qs0);
qs1qs0 = _mm_srli_si128(qs1qs0, 4);
*(int *)(s + 5 * p - 2) = _mm_cvtsi128_si32(qs1qs0);
......@@ -287,6 +299,19 @@ void aom_lpf_vertical_4_sse2(uint8_t *s, int p /* pitch */,
*(int *)(s + 6 * p - 2) = _mm_cvtsi128_si32(qs1qs0);
qs1qs0 = _mm_srli_si128(qs1qs0, 4);
*(int *)(s + 7 * p - 2) = _mm_cvtsi128_si32(qs1qs0);
#endif
}
static INLINE void store_buffer_horz_8(const __m128i *x, int p, int num,
uint8_t *s) {
#if CONFIG_PARALLEL_DEBLOCKING
*(int32_t *)(s - (num + 1) * p) = _mm_cvtsi128_si32(*x);
const __m128i hi = _mm_srli_si128(*x, 8);
*(int32_t *)(s + num * p) = _mm_cvtsi128_si32(hi);
#else
_mm_storel_epi64((__m128i *)(s - (num + 1) * p), *x);
_mm_storeh_pi((__m64 *)(s + num * p), _mm_castsi128_ps(*x));
#endif
}
void aom_lpf_horizontal_edge_8_sse2(unsigned char *s, int p,
......@@ -580,44 +605,37 @@ void aom_lpf_horizontal_edge_8_sse2(unsigned char *s, int p,
q6p6 = _mm_andnot_si128(flat2, q6p6);
flat2_q6p6 = _mm_and_si128(flat2, flat2_q6p6);
q6p6 = _mm_or_si128(q6p6, flat2_q6p6);
_mm_storel_epi64((__m128i *)(s - 7 * p), q6p6);
_mm_storeh_pi((__m64 *)(s + 6 * p), _mm_castsi128_ps(q6p6));
store_buffer_horz_8(&q6p6, p, 6, s);
q5p5 = _mm_andnot_si128(flat2, q5p5);
flat2_q5p5 = _mm_and_si128(flat2, flat2_q5p5);
q5p5 = _mm_or_si128(q5p5, flat2_q5p5);
_mm_storel_epi64((__m128i *)(s - 6 * p), q5p5);
_mm_storeh_pi((__m64 *)(s + 5 * p), _mm_castsi128_ps(q5p5));
store_buffer_horz_8(&q5p5, p, 5, s);
q4p4 = _mm_andnot_si128(flat2, q4p4);
flat2_q4p4 = _mm_and_si128(flat2, flat2_q4p4);
q4p4 = _mm_or_si128(q4p4, flat2_q4p4);
_mm_storel_epi64((__m128i *)(s - 5 * p), q4p4);
_mm_storeh_pi((__m64 *)(s + 4 * p), _mm_castsi128_ps(q4p4));
store_buffer_horz_8(&q4p4, p, 4, s);
q3p3 = _mm_andnot_si128(flat2, q3p3);
flat2_q3p3 = _mm_and_si128(flat2, flat2_q3p3);
q3p3 = _mm_or_si128(q3p3, flat2_q3p3);
_mm_storel_epi64((__m128i *)(s - 4 * p), q3p3);
_mm_storeh_pi((__m64 *)(s + 3 * p), _mm_castsi128_ps(q3p3));
store_buffer_horz_8(&q3p3, p, 3, s);
q2p2 = _mm_andnot_si128(flat2, q2p2);
flat2_q2p2 = _mm_and_si128(flat2, flat2_q2p2);
q2p2 = _mm_or_si128(q2p2, flat2_q2p2);
_mm_storel_epi64((__m128i *)(s - 3 * p), q2p2);
_mm_storeh_pi((__m64 *)(s + 2 * p), _mm_castsi128_ps(q2p2));
store_buffer_horz_8(&q2p2, p, 2, s);
q1p1 = _mm_andnot_si128(flat2, q1p1);
flat2_q1p1 = _mm_and_si128(flat2, flat2_q1p1);
q1p1 = _mm_or_si128(q1p1, flat2_q1p1);
_mm_storel_epi64((__m128i *)(s - 2 * p), q1p1);
_mm_storeh_pi((__m64 *)(s + 1 * p), _mm_castsi128_ps(q1p1));
store_buffer_horz_8(&q1p1, p, 1, s);
q0p0 = _mm_andnot_si128(flat2, q0p0);
flat2_q0p0 = _mm_and_si128(flat2, flat2_q0p0);
q0p0 = _mm_or_si128(q0p0, flat2_q0p0);
_mm_storel_epi64((__m128i *)(s - 1 * p), q0p0);
_mm_storeh_pi((__m64 *)(s - 0 * p), _mm_castsi128_ps(q0p0));
store_buffer_horz_8(&q0p0, p, 0, s);
}
}
......@@ -651,10 +669,33 @@ static INLINE __m128i filter16_mask(const __m128i *const flat,
return _mm_or_si128(_mm_andnot_si128(*flat, *other_filt), result);
}
void aom_lpf_horizontal_edge_16_sse2(unsigned char *s, int p,
const unsigned char *_blimit,
const unsigned char *_limit,
const unsigned char *_thresh) {
typedef enum { FOUR_PIXELS, EIGHT_PIXELS, SIXTEEN_PIXELS } PixelOutput;
static INLINE void store_buffer_horz_16(PixelOutput pixel_num, const __m128i *x,
int p, int offset, uint8_t *s) {
int i;
if (pixel_num == FOUR_PIXELS) {
for (i = 13; i >= 0; i--) {
*(int32_t *)(s - (i - offset) * p) = _mm_cvtsi128_si32(x[i]);
}
}
if (pixel_num == EIGHT_PIXELS) {
for (i = 13; i >= 0; i--) {
_mm_storel_epi64((__m128i *)(s - (i - offset) * p), x[i]);
}
}
if (pixel_num == SIXTEEN_PIXELS) {
for (i = 13; i >= 0; i--) {
_mm_storeu_si128((__m128i *)(s - (i - offset) * p), x[i]);
}
}
}
static INLINE void lpf_horz_edge_16_internal(PixelOutput pixel_num,
unsigned char *s, int p,
const unsigned char *_blimit,
const unsigned char *_limit,
const unsigned char *_thresh) {
const __m128i zero = _mm_set1_epi16(0);
const __m128i one = _mm_set1_epi8(1);
const __m128i blimit = _mm_load_si128((const __m128i *)_blimit);
......@@ -910,73 +951,62 @@ void aom_lpf_horizontal_edge_16_sse2(unsigned char *s, int p,
f_hi = _mm_add_epi16(_mm_add_epi16(p0_hi, q0_hi), f_hi);
f_hi = _mm_add_epi16(_mm_add_epi16(p5_hi, eight), f_hi);
p6 = filter16_mask(&flat2, &p6, &f_lo, &f_hi);
_mm_storeu_si128((__m128i *)(s - 7 * p), p6);
__m128i x[14];
x[13] = filter16_mask(&flat2, &p6, &f_lo, &f_hi);
f_lo = filter_add2_sub2(&f_lo, &q1_lo, &p5_lo, &p6_lo, &p7_lo);
f_hi = filter_add2_sub2(&f_hi, &q1_hi, &p5_hi, &p6_hi, &p7_hi);
p5 = filter16_mask(&flat2, &p5, &f_lo, &f_hi);
_mm_storeu_si128((__m128i *)(s - 6 * p), p5);
x[12] = filter16_mask(&flat2, &p5, &f_lo, &f_hi);
f_lo = filter_add2_sub2(&f_lo, &q2_lo, &p4_lo, &p5_lo, &p7_lo);
f_hi = filter_add2_sub2(&f_hi, &q2_hi, &p4_hi, &p5_hi, &p7_hi);
p4 = filter16_mask(&flat2, &p4, &f_lo, &f_hi);
_mm_storeu_si128((__m128i *)(s - 5 * p), p4);
x[11] = filter16_mask(&flat2, &p4, &f_lo, &f_hi);
f_lo = filter_add2_sub2(&f_lo, &q3_lo, &p3_lo, &p4_lo, &p7_lo);
f_hi = filter_add2_sub2(&f_hi, &q3_hi, &p3_hi, &p4_hi, &p7_hi);
p3 = filter16_mask(&flat2, &p3, &f_lo, &f_hi);
_mm_storeu_si128((__m128i *)(s - 4 * p), p3);
x[10] = filter16_mask(&flat2, &p3, &f_lo, &f_hi);
f_lo = filter_add2_sub2(&f_lo, &q4_lo, &p2_lo, &p3_lo, &p7_lo);
f_hi = filter_add2_sub2(&f_hi, &q4_hi, &p2_hi, &p3_hi, &p7_hi);
op2 = filter16_mask(&flat2, &op2, &f_lo, &f_hi);
_mm_storeu_si128((__m128i *)(s - 3 * p), op2);
x[9] = filter16_mask(&flat2, &op2, &f_lo, &f_hi);
f_lo = filter_add2_sub2(&f_lo, &q5_lo, &p1_lo, &p2_lo, &p7_lo);
f_hi = filter_add2_sub2(&f_hi, &q5_hi, &p1_hi, &p2_hi, &p7_hi);
op1 = filter16_mask(&flat2, &op1, &f_lo, &f_hi);
_mm_storeu_si128((__m128i *)(s - 2 * p), op1);
x[8] = filter16_mask(&flat2, &op1, &f_lo, &f_hi);
f_lo = filter_add2_sub2(&f_lo, &q6_lo, &p0_lo, &p1_lo, &p7_lo);
f_hi = filter_add2_sub2(&f_hi, &q6_hi, &p0_hi, &p1_hi, &p7_hi);
op0 = filter16_mask(&flat2, &op0, &f_lo, &f_hi);
_mm_storeu_si128((__m128i *)(s - 1 * p), op0);
x[7] = filter16_mask(&flat2, &op0, &f_lo, &f_hi);
f_lo = filter_add2_sub2(&f_lo, &q7_lo, &q0_lo, &p0_lo, &p7_lo);
f_hi = filter_add2_sub2(&f_hi, &q7_hi, &q0_hi, &p0_hi, &p7_hi);
oq0 = filter16_mask(&flat2, &oq0, &f_lo, &f_hi);
_mm_storeu_si128((__m128i *)(s - 0 * p), oq0);
x[6] = filter16_mask(&flat2, &oq0, &f_lo, &f_hi);
f_lo = filter_add2_sub2(&f_lo, &q7_lo, &q1_lo, &p6_lo, &q0_lo);
f_hi = filter_add2_sub2(&f_hi, &q7_hi, &q1_hi, &p6_hi, &q0_hi);
oq1 = filter16_mask(&flat2, &oq1, &f_lo, &f_hi);
_mm_storeu_si128((__m128i *)(s + 1 * p), oq1);
x[5] = filter16_mask(&flat2, &oq1, &f_lo, &f_hi);
f_lo = filter_add2_sub2(&f_lo, &q7_lo, &q2_lo, &p5_lo, &q1_lo);
f_hi = filter_add2_sub2(&f_hi, &q7_hi, &q2_hi, &p5_hi, &q1_hi);
oq2 = filter16_mask(&flat2, &oq2, &f_lo, &f_hi);
_mm_storeu_si128((__m128i *)(s + 2 * p), oq2);
x[4] = filter16_mask(&flat2, &oq2, &f_lo, &f_hi);
f_lo = filter_add2_sub2(&f_lo, &q7_lo, &q3_lo, &p4_lo, &q2_lo);
f_hi = filter_add2_sub2(&f_hi, &q7_hi, &q3_hi, &p4_hi, &q2_hi);
q3 = filter16_mask(&flat2, &q3, &f_lo, &f_hi);
_mm_storeu_si128((__m128i *)(s + 3 * p), q3);
x[3] = filter16_mask(&flat2, &q3, &f_lo, &f_hi);
f_lo = filter_add2_sub2(&f_lo, &q7_lo, &q4_lo, &p3_lo, &q3_lo);
f_hi = filter_add2_sub2(&f_hi, &q7_hi, &q4_hi, &p3_hi, &q3_hi);
q4 = filter16_mask(&flat2, &q4, &f_lo, &f_hi);
_mm_storeu_si128((__m128i *)(s + 4 * p), q4);
x[2] = filter16_mask(&flat2, &q4, &f_lo, &f_hi);
f_lo = filter_add2_sub2(&f_lo, &q7_lo, &q5_lo, &p2_lo, &q4_lo);
f_hi = filter_add2_sub2(&f_hi, &q7_hi, &q5_hi, &p2_hi, &q4_hi);
q5 = filter16_mask(&flat2, &q5, &f_lo, &f_hi);
_mm_storeu_si128((__m128i *)(s + 5 * p), q5);
x[1] = filter16_mask(&flat2, &q5, &f_lo, &f_hi);
f_lo = filter_add2_sub2(&f_lo, &q7_lo, &q6_lo, &p1_lo, &q5_lo);
f_hi = filter_add2_sub2(&f_hi, &q7_hi, &q6_hi, &p1_hi, &q5_hi);
q6 = filter16_mask(&flat2, &q6, &f_lo, &f_hi);
_mm_storeu_si128((__m128i *)(s + 6 * p), q6);
x[0] = filter16_mask(&flat2, &q6, &f_lo, &f_hi);
store_buffer_horz_16(pixel_num, x, p, 6, s);
}
// wide flat
// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
......@@ -1186,15 +1216,35 @@ void aom_lpf_horizontal_8_sse2(unsigned char *s, int p,
p2 = _mm_and_si128(flat, p2);
p2 = _mm_or_si128(work_a, p2);
#if CONFIG_PARALLEL_DEBLOCKING
*(int32_t *)(s - 3 * p) = _mm_cvtsi128_si32(p2);
*(int32_t *)(s - 2 * p) = _mm_cvtsi128_si32(p1);
*(int32_t *)(s - 1 * p) = _mm_cvtsi128_si32(p0);
*(int32_t *)(s + 0 * p) = _mm_cvtsi128_si32(q0);
*(int32_t *)(s + 1 * p) = _mm_cvtsi128_si32(q1);
*(int32_t *)(s + 2 * p) = _mm_cvtsi128_si32(q2);
#else
_mm_storel_epi64((__m128i *)(s - 3 * p), p2);
_mm_storel_epi64((__m128i *)(s - 2 * p), p1);
_mm_storel_epi64((__m128i *)(s - 1 * p), p0);
_mm_storel_epi64((__m128i *)(s + 0 * p), q0);
_mm_storel_epi64((__m128i *)(s + 1 * p), q1);
_mm_storel_epi64((__m128i *)(s + 2 * p), q2);
#endif
}
}
void aom_lpf_horizontal_edge_16_sse2(unsigned char *s, int p,
const unsigned char *_blimit,
const unsigned char *_limit,
const unsigned char *_thresh) {
#if CONFIG_PARALLEL_DEBLOCKING
lpf_horz_edge_16_internal(FOUR_PIXELS, s, p, _blimit, _limit, _thresh);
#else
lpf_horz_edge_16_internal(SIXTEEN_PIXELS, s, p, _blimit, _limit, _thresh);
#endif
}
void aom_lpf_horizontal_8_dual_sse2(uint8_t *s, int p, const uint8_t *_blimit0,
const uint8_t *_limit0,
const uint8_t *_thresh0,
......
......@@ -13,13 +13,12 @@
#include "./aom_config.h"
#include "./aom_dsp_rtcd.h"
#include "av1/common/av1_loopfilter.h"
#include "av1/common/onyxc_int.h"
#include "av1/common/reconinter.h"
#include "aom_dsp/aom_dsp_common.h"
#include "aom_mem/aom_mem.h"
#include "aom_ports/mem.h"
#include "av1/common/av1_loopfilter.h"
#include "av1/common/onyxc_int.h"
#include "av1/common/reconinter.h"
#include "av1/common/seg_common.h"
#if CONFIG_LPF_DIRECT
......@@ -3002,8 +3001,8 @@ static void av1_filter_block_plane_vert(
params.hev_thr, cm->bit_depth);
else
#endif // CONFIG_HIGHBITDEPTH
aom_lpf_vertical_4_c(filt_start, line_length, params.mblim,
params.lim, params.hev_thr);
aom_lpf_vertical_4(filt_start, line_length, params.mblim,
params.lim, params.hev_thr);
break;
// apply 8-tap filtering
case 8:
......@@ -3014,8 +3013,8 @@ static void av1_filter_block_plane_vert(
params.hev_thr, cm->bit_depth);
else
#endif // CONFIG_HIGHBITDEPTH
aom_lpf_vertical_8_c(filt_start, line_length, params.mblim,
params.lim, params.hev_thr);
aom_lpf_vertical_8(filt_start, line_length, params.mblim,
params.lim, params.hev_thr);
break;
// apply 16-tap filtering
case 16:
......@@ -3026,8 +3025,8 @@ static void av1_filter_block_plane_vert(
params.hev_thr, cm->bit_depth);
else
#endif // CONFIG_HIGHBITDEPTH
aom_lpf_vertical_16_c(filt_start, line_length, params.mblim,
params.lim, params.hev_thr);
aom_lpf_vertical_16(filt_start, line_length, params.mblim,
params.lim, params.hev_thr);
break;
// no filtering
default: break;
......@@ -3060,8 +3059,8 @@ static void av1_filter_block_plane_vert(
params.hev_thr, cm->bit_depth);
else
#endif // CONFIG_HIGHBITDEPTH
aom_lpf_vertical_4_c(filt_start, line_length, params.mblim,
params.lim, params.hev_thr);
aom_lpf_vertical_4(filt_start, line_length, params.mblim, params.lim,
params.hev_thr);
for (i = 0; i < 128; ++i) {
if (orig_pos[i] >= 0) src[orig_pos[i]] = block[i];
......@@ -3078,8 +3077,8 @@ static void av1_filter_block_plane_vert(
cm->bit_depth);
else
#endif // CONFIG_HIGHBITDEPTH
aom_lpf_vertical_4_c(p, dst_stride, params.mblim, params.lim,
params.hev_thr);
aom_lpf_vertical_4(p, dst_stride, params.mblim, params.lim,
params.hev_thr);
break;
// apply 8-tap filtering
case 8:
......@@ -3090,8 +3089,8 @@ static void av1_filter_block_plane_vert(
cm->bit_depth);
else
#endif // CONFIG_HIGHBITDEPTH
aom_lpf_vertical_8_c(p, dst_stride, params.mblim, params.lim,
params.hev_thr);
aom_lpf_vertical_8(p, dst_stride, params.mblim, params.lim,
params.hev_thr);
break;
// apply 16-tap filtering
case 16:
......@@ -3102,8 +3101,8 @@ static void av1_filter_block_plane_vert(
cm->bit_depth);
else
#endif // CONFIG_HIGHBITDEPTH
aom_lpf_vertical_16_c(p, dst_stride, params.mblim, params.lim,
params.hev_thr);
aom_lpf_vertical_16(p, dst_stride, params.mblim, params.lim,
params.hev_thr);