Commit 8cf6f156 authored by Angie Chiang's avatar Angie Chiang

Move simd optimization of 10/12 taps convolve function to AOM

By turning on ext_interp experiment
Encoder time is reduced for 26.7%
Decoder time is reduced for 15.4%

Change-Id: I14e197dd7ed3261339b0643fa4ed0947f74274fc
parent 30abc082
......@@ -31,6 +31,13 @@ AV1_COMMON_SRCS-yes += common/filter.h
AV1_COMMON_SRCS-yes += common/filter.c
AV1_COMMON_SRCS-yes += common/convolve.c
AV1_COMMON_SRCS-yes += common/convolve.h
AV1_COMMON_SRCS-$(HAVE_SSSE3) += common/x86/av1_convolve_ssse3.c
AV1_COMMON_SRCS-$(HAVE_SSSE3) += common/x86/av1_convolve_filters_ssse3.h
ifeq ($(CONFIG_AOM_HIGHBITDEPTH),yes)
AV1_COMMON_SRCS-$(HAVE_SSE4_1) += common/x86/av1_highbd_convolve_sse4.c
AV1_COMMON_SRCS-$(HAVE_SSE4_1) += common/x86/av1_highbd_convolve_filters_sse4.h
endif
AV1_COMMON_SRCS-yes += common/idct.h
AV1_COMMON_SRCS-yes += common/idct.c
AV1_COMMON_SRCS-yes += common/av1_inv_txfm.h
......
......@@ -17,6 +17,7 @@ struct search_site_config;
struct mv;
union int_mv;
struct yv12_buffer_config;
struct InterpFilterParams;
EOF
}
forward_decls qw/av1_common_forward_decls/;
......@@ -54,6 +55,19 @@ if ($opts{arch} eq "x86_64") {
$avx2_x86_64 = 'avx2';
}
add_proto qw/void av1_convolve_horiz/, "const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const struct InterpFilterParams fp, const int subpel_x_q4, int x_step_q4, int avg";
specialize qw/av1_convolve_horiz ssse3/;
add_proto qw/void av1_convolve_vert/, "const uint8_t *src, int src_stride, uint8_t *dst, int dst_stride, int w, int h, const struct InterpFilterParams fp, const int subpel_x_q4, int x_step_q4, int avg";
specialize qw/av1_convolve_vert ssse3/;
if (aom_config("CONFIG_AOM_HIGHBITDEPTH") eq "yes") {
add_proto qw/void av1_highbd_convolve_horiz/, "const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const struct InterpFilterParams fp, const int subpel_x_q4, int x_step_q4, int avg, int bd";
specialize qw/av1_highbd_convolve_horiz sse4_1/;
add_proto qw/void av1_highbd_convolve_vert/, "const uint16_t *src, int src_stride, uint16_t *dst, int dst_stride, int w, int h, const struct InterpFilterParams fp, const int subpel_x_q4, int x_step_q4, int avg, int bd";
specialize qw/av1_highbd_convolve_vert sse4_1/;
}
#
# dct
#
......
......@@ -12,6 +12,7 @@
#include <assert.h>
#include <string.h>
#include "./av1_rtcd.h"
#include "av1/common/filter.h"
#include "av1/common/enums.h"
#include "aom_dsp/aom_dsp_common.h"
......@@ -22,10 +23,10 @@
#define MAX_STEP (32)
#define MAX_FILTER_TAP (12)
static void convolve_horiz(const uint8_t *src, int src_stride, uint8_t *dst,
int dst_stride, int w, int h,
const InterpFilterParams filter_params,
const int subpel_x_q4, int x_step_q4, int avg) {
void av1_convolve_horiz_c(const uint8_t *src, int src_stride, uint8_t *dst,
int dst_stride, int w, int h,
const InterpFilterParams filter_params,
const int subpel_x_q4, int x_step_q4, int avg) {
int x, y;
int filter_size = filter_params.taps;
src -= filter_size / 2 - 1;
......@@ -50,10 +51,10 @@ static void convolve_horiz(const uint8_t *src, int src_stride, uint8_t *dst,
}
}
static void convolve_vert(const uint8_t *src, int src_stride, uint8_t *dst,
int dst_stride, int w, int h,
const InterpFilterParams filter_params,
const int subpel_y_q4, int y_step_q4, int avg) {
void av1_convolve_vert_c(const uint8_t *src, int src_stride, uint8_t *dst,
int dst_stride, int w, int h,
const InterpFilterParams filter_params,
const int subpel_y_q4, int y_step_q4, int avg) {
int x, y;
int filter_size = filter_params.taps;
src -= src_stride * (filter_size / 2 - 1);
......@@ -121,13 +122,13 @@ void av1_convolve(const uint8_t *src, int src_stride, uint8_t *dst,
} else if (ignore_vert) {
InterpFilterParams filter_params = get_interp_filter_params(*interp_filter);
assert(filter_params.taps <= MAX_FILTER_TAP);
convolve_horiz(src, src_stride, dst, dst_stride, w, h, filter_params,
subpel_x_q4, x_step_q4, ref_idx);
av1_convolve_horiz(src, src_stride, dst, dst_stride, w, h, filter_params,
subpel_x_q4, x_step_q4, ref_idx);
} else if (ignore_horiz) {
InterpFilterParams filter_params = get_interp_filter_params(*interp_filter);
assert(filter_params.taps <= MAX_FILTER_TAP);
convolve_vert(src, src_stride, dst, dst_stride, w, h, filter_params,
subpel_y_q4, y_step_q4, ref_idx);
av1_convolve_vert(src, src_stride, dst, dst_stride, w, h, filter_params,
subpel_y_q4, y_step_q4, ref_idx);
} else {
// temp's size is set to (maximum possible intermediate_height) *
// MAX_BLOCK_WIDTH
......@@ -142,22 +143,22 @@ void av1_convolve(const uint8_t *src, int src_stride, uint8_t *dst,
assert(filter_size <= MAX_FILTER_TAP);
convolve_horiz(src - src_stride * (filter_size / 2 - 1), src_stride, temp,
temp_stride, w, intermediate_height, filter_params,
subpel_x_q4, x_step_q4, 0);
av1_convolve_horiz(src - src_stride * (filter_size / 2 - 1), src_stride,
temp, temp_stride, w, intermediate_height, filter_params,
subpel_x_q4, x_step_q4, 0);
convolve_vert(temp + temp_stride * (filter_size / 2 - 1), temp_stride, dst,
dst_stride, w, h, filter_params, subpel_y_q4, y_step_q4,
ref_idx);
av1_convolve_vert(temp + temp_stride * (filter_size / 2 - 1), temp_stride,
dst, dst_stride, w, h, filter_params, subpel_y_q4,
y_step_q4, ref_idx);
}
}
#if CONFIG_AOM_HIGHBITDEPTH
static void highbd_convolve_horiz(const uint16_t *src, int src_stride,
uint16_t *dst, int dst_stride, int w, int h,
const InterpFilterParams filter_params,
const int subpel_x_q4, int x_step_q4, int avg,
int bd) {
void av1_highbd_convolve_horiz_c(const uint16_t *src, int src_stride,
uint16_t *dst, int dst_stride, int w, int h,
const InterpFilterParams filter_params,
const int subpel_x_q4, int x_step_q4, int avg,
int bd) {
int x, y;
int filter_size = filter_params.taps;
src -= filter_size / 2 - 1;
......@@ -183,11 +184,11 @@ static void highbd_convolve_horiz(const uint16_t *src, int src_stride,
}
}
static void highbd_convolve_vert(const uint16_t *src, int src_stride,
uint16_t *dst, int dst_stride, int w, int h,
const InterpFilterParams filter_params,
const int subpel_y_q4, int y_step_q4, int avg,
int bd) {
void av1_highbd_convolve_vert_c(const uint16_t *src, int src_stride,
uint16_t *dst, int dst_stride, int w, int h,
const InterpFilterParams filter_params,
const int subpel_y_q4, int y_step_q4, int avg,
int bd) {
int x, y;
int filter_size = filter_params.taps;
src -= src_stride * (filter_size / 2 - 1);
......@@ -259,12 +260,14 @@ void av1_highbd_convolve(const uint8_t *src8, int src_stride, uint8_t *dst8,
highbd_convolve_copy(src, src_stride, dst, dst_stride, w, h, ref_idx, bd);
} else if (ignore_vert) {
InterpFilterParams filter_params = get_interp_filter_params(*interp_filter);
highbd_convolve_horiz(src, src_stride, dst, dst_stride, w, h, filter_params,
subpel_x_q4, x_step_q4, ref_idx, bd);
av1_highbd_convolve_horiz(src, src_stride, dst, dst_stride, w, h,
filter_params, subpel_x_q4, x_step_q4, ref_idx,
bd);
} else if (ignore_horiz) {
InterpFilterParams filter_params = get_interp_filter_params(*interp_filter);
highbd_convolve_vert(src, src_stride, dst, dst_stride, w, h, filter_params,
subpel_y_q4, y_step_q4, ref_idx, bd);
av1_highbd_convolve_vert(src, src_stride, dst, dst_stride, w, h,
filter_params, subpel_y_q4, y_step_q4, ref_idx,
bd);
} else {
// temp's size is set to (maximum possible intermediate_height) *
// MAX_BLOCK_WIDTH
......@@ -279,13 +282,13 @@ void av1_highbd_convolve(const uint8_t *src8, int src_stride, uint8_t *dst8,
(((h - 1) * y_step_q4 + subpel_y_q4) >> SUBPEL_BITS) + filter_size;
assert(filter_size <= MAX_FILTER_TAP);
highbd_convolve_horiz(src - src_stride * (filter_size / 2 - 1), src_stride,
temp, temp_stride, w, intermediate_height,
filter_params, subpel_x_q4, x_step_q4, 0, bd);
av1_highbd_convolve_horiz(
src - src_stride * (filter_size / 2 - 1), src_stride, temp, temp_stride,
w, intermediate_height, filter_params, subpel_x_q4, x_step_q4, 0, bd);
highbd_convolve_vert(temp + temp_stride * (filter_size / 2 - 1),
temp_stride, dst, dst_stride, w, h, filter_params,
subpel_y_q4, y_step_q4, ref_idx, bd);
av1_highbd_convolve_vert(temp + temp_stride * (filter_size / 2 - 1),
temp_stride, dst, dst_stride, w, h, filter_params,
subpel_y_q4, y_step_q4, ref_idx, bd);
}
}
#endif // CONFIG_AOM_HIGHBITDEPTH
......@@ -141,25 +141,33 @@ const InterpKernel *av1_filter_kernels[4] = { sub_pel_filters_8,
sub_pel_filters_8sharp,
bilinear_filters };
#if CONFIG_EXT_INTERP
static const InterpFilterParams
interp_filter_params_list[SWITCHABLE_FILTERS + 1] = {
{ (const int16_t *)sub_pel_filters_8, SUBPEL_TAPS, SUBPEL_SHIFTS },
{ (const int16_t *)sub_pel_filters_8smooth, SUBPEL_TAPS, SUBPEL_SHIFTS },
{ (const int16_t *)sub_pel_filters_10sharp, 10, SUBPEL_SHIFTS },
{ (const int16_t *)sub_pel_filters_8smooth2, SUBPEL_TAPS, SUBPEL_SHIFTS },
{ (const int16_t *)sub_pel_filters_12sharp, 12, SUBPEL_SHIFTS },
{ (const int16_t *)bilinear_filters, SUBPEL_TAPS, SUBPEL_SHIFTS }
};
static const InterpFilterParams interp_filter_params_list[SWITCHABLE_FILTERS +
1] = {
{ (const int16_t *)sub_pel_filters_8, SUBPEL_TAPS, SUBPEL_SHIFTS, EIGHTTAP },
{ (const int16_t *)sub_pel_filters_8smooth, SUBPEL_TAPS, SUBPEL_SHIFTS,
EIGHTTAP_SMOOTH },
{ (const int16_t *)sub_pel_filters_10sharp, 10, SUBPEL_SHIFTS,
MULTITAP_SHARP },
{ (const int16_t *)sub_pel_filters_8smooth2, SUBPEL_TAPS, SUBPEL_SHIFTS,
EIGHTTAP_SMOOTH2 },
{ (const int16_t *)sub_pel_filters_12sharp, 12, SUBPEL_SHIFTS,
MULTITAP_SHARP2 },
{ (const int16_t *)bilinear_filters, SUBPEL_TAPS, SUBPEL_SHIFTS, BILINEAR }
};
#else // CONFIG_EXT_INTERP
static const InterpFilterParams
interp_filter_params_list[SWITCHABLE_FILTERS + 1] = {
{ (const int16_t *)sub_pel_filters_8, SUBPEL_TAPS, SUBPEL_SHIFTS },
{ (const int16_t *)sub_pel_filters_8smooth, SUBPEL_TAPS, SUBPEL_SHIFTS },
{ (const int16_t *)sub_pel_filters_8sharp, SUBPEL_TAPS, SUBPEL_SHIFTS },
{ (const int16_t *)bilinear_filters, SUBPEL_TAPS, SUBPEL_SHIFTS }
};
static const InterpFilterParams interp_filter_params_list[SWITCHABLE_FILTERS +
1] = {
{ (const int16_t *)sub_pel_filters_8, SUBPEL_TAPS, SUBPEL_SHIFTS, EIGHTTAP },
{ (const int16_t *)sub_pel_filters_8smooth, SUBPEL_TAPS, SUBPEL_SHIFTS,
EIGHTTAP_SMOOTH },
{ (const int16_t *)sub_pel_filters_8sharp, SUBPEL_TAPS, SUBPEL_SHIFTS,
EIGHTTAP_SHARP },
{ (const int16_t *)bilinear_filters, SUBPEL_TAPS, SUBPEL_SHIFTS, BILINEAR }
};
#endif // CONFIG_EXT_INTERP
InterpFilterParams get_interp_filter_params(InterpFilter interp_filter) {
return interp_filter_params_list[interp_filter];
InterpFilterParams params = interp_filter_params_list[interp_filter];
assert(params.interp_filter == interp_filter);
return params;
}
......@@ -57,6 +57,7 @@ typedef struct InterpFilterParams {
const int16_t* filter_ptr;
uint16_t taps;
uint16_t subpel_shifts;
InterpFilter interp_filter;
} InterpFilterParams;
static INLINE const int16_t* get_interp_filter_subpel_kernel(
......
This diff is collapsed.
This diff is collapsed.
/*
* Copyright (c) 2016, Alliance for Open Media. All rights reserved
*
* This source code is subject to the terms of the BSD 2 Clause License and
* the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
* was not distributed with this source code in the LICENSE file, you can
* obtain it at www.aomedia.org/license/software. If the Alliance for Open
* Media Patent License 1.0 was not distributed with this source code in the
* PATENTS file, you can obtain it at www.aomedia.org/license/patent.
*/
#ifndef AV1_COMMON_X86_AV1_HIGHBD_CONVOLVE_FILTERS_SSE4_H_
#define AV1_COMMON_X86_AV1_HIGHBD_CONVOLVE_FILTERS_SSE4_H_
#include "./aom_config.h"
#include "av1/common/filter.h"
#if CONFIG_AOM_HIGHBITDEPTH
#if CONFIG_EXT_INTERP
DECLARE_ALIGNED(16, static const int16_t,
sub_pel_filters_10sharp_highbd_ver_signal_dir[15][6][8]) = {
{
{
0, 0, 0, 0, 0, 0, 0, 0,
},
{
-1, 3, -1, 3, -1, 3, -1, 3,
},
{
-6, 127, -6, 127, -6, 127, -6, 127,
},
{
8, -4, 8, -4, 8, -4, 8, -4,
},
{
2, -1, 2, -1, 2, -1, 2, -1,
},
{
0, 0, 0, 0, 0, 0, 0, 0,
},
},
{
{
0, 1, 0, 1, 0, 1, 0, 1,
},
{
-2, 5, -2, 5, -2, 5, -2, 5,
},
{
-12, 124, -12, 124, -12, 124, -12, 124,
},
{
18, -7, 18, -7, 18, -7, 18, -7,
},
{
3, -2, 3, -2, 3, -2, 3, -2,
},
{
0, 0, 0, 0, 0, 0, 0, 0,
},
},
{
{
0, 1, 0, 1, 0, 1, 0, 1,
},
{
-3, 7, -3, 7, -3, 7, -3, 7,
},
{
-17, 119, -17, 119, -17, 119, -17, 119,
},
{
28, -11, 28, -11, 28, -11, 28, -11,
},
{
5, -2, 5, -2, 5, -2, 5, -2,
},
{
1, 0, 1, 0, 1, 0, 1, 0,
},
},
{
{
0, 1, 0, 1, 0, 1, 0, 1,
},
{
-4, 8, -4, 8, -4, 8, -4, 8,
},
{
-20, 114, -20, 114, -20, 114, -20, 114,
},
{
38, -14, 38, -14, 38, -14, 38, -14,
},
{
7, -3, 7, -3, 7, -3, 7, -3,
},
{
1, 0, 1, 0, 1, 0, 1, 0,
},
},
{
{
0, 1, 0, 1, 0, 1, 0, 1,
},
{
-4, 9, -4, 9, -4, 9, -4, 9,
},
{
-22, 107, -22, 107, -22, 107, -22, 107,
},
{
49, -17, 49, -17, 49, -17, 49, -17,
},
{
8, -4, 8, -4, 8, -4, 8, -4,
},
{
1, 0, 1, 0, 1, 0, 1, 0,
},
},
{
{
0, 2, 0, 2, 0, 2, 0, 2,
},
{
-5, 10, -5, 10, -5, 10, -5, 10,
},
{
-24, 99, -24, 99, -24, 99, -24, 99,
},
{
59, -20, 59, -20, 59, -20, 59, -20,
},
{
9, -4, 9, -4, 9, -4, 9, -4,
},
{
2, 0, 2, 0, 2, 0, 2, 0,
},
},
{
{
0, 2, 0, 2, 0, 2, 0, 2,
},
{
-5, 10, -5, 10, -5, 10, -5, 10,
},
{
-24, 90, -24, 90, -24, 90, -24, 90,
},
{
70, -22, 70, -22, 70, -22, 70, -22,
},
{
10, -5, 10, -5, 10, -5, 10, -5,
},
{
2, 0, 2, 0, 2, 0, 2, 0,
},
},
{
{
0, 2, 0, 2, 0, 2, 0, 2,
},
{
-5, 10, -5, 10, -5, 10, -5, 10,
},
{
-23, 80, -23, 80, -23, 80, -23, 80,
},
{
80, -23, 80, -23, 80, -23, 80, -23,
},
{
10, -5, 10, -5, 10, -5, 10, -5,
},
{
2, 0, 2, 0, 2, 0, 2, 0,
},
},
{
{
0, 2, 0, 2, 0, 2, 0, 2,
},
{
-5, 10, -5, 10, -5, 10, -5, 10,
},
{
-22, 70, -22, 70, -22, 70, -22, 70,
},
{
90, -24, 90, -24, 90, -24, 90, -24,
},
{
10, -5, 10, -5, 10, -5, 10, -5,
},
{
2, 0, 2, 0, 2, 0, 2, 0,
},
},
{
{
0, 2, 0, 2, 0, 2, 0, 2,
},
{
-4, 9, -4, 9, -4, 9, -4, 9,
},
{
-20, 59, -20, 59, -20, 59, -20, 59,
},
{
99, -24, 99, -24, 99, -24, 99, -24,
},
{
10, -5, 10, -5, 10, -5, 10, -5,
},
{
2, 0, 2, 0, 2, 0, 2, 0,
},
},
{
{
0, 1, 0, 1, 0, 1, 0, 1,
},
{
-4, 8, -4, 8, -4, 8, -4, 8,
},
{
-17, 49, -17, 49, -17, 49, -17, 49,
},
{
107, -22, 107, -22, 107, -22, 107, -22,
},
{
9, -4, 9, -4, 9, -4, 9, -4,
},
{
1, 0, 1, 0, 1, 0, 1, 0,
},
},
{
{
0, 1, 0, 1, 0, 1, 0, 1,
},
{
-3, 7, -3, 7, -3, 7, -3, 7,
},
{
-14, 38, -14, 38, -14, 38, -14, 38,
},
{
114, -20, 114, -20, 114, -20, 114, -20,
},
{
8, -4, 8, -4, 8, -4, 8, -4,
},
{
1, 0, 1, 0, 1, 0, 1, 0,
},
},
{
{
0, 1, 0, 1, 0, 1, 0, 1,
},
{
-2, 5, -2, 5, -2, 5, -2, 5,
},
{
-11, 28, -11, 28, -11, 28, -11, 28,
},
{
119, -17, 119, -17, 119, -17, 119, -17,
},
{
7, -3, 7, -3, 7, -3, 7, -3,
},
{
1, 0, 1, 0, 1, 0, 1, 0,
},
},
{
{
0, 0, 0, 0, 0, 0, 0, 0,
},
{
-2, 3, -2, 3, -2, 3, -2, 3,
},
{
-7, 18, -7, 18, -7, 18, -7, 18,
},
{
124, -12, 124, -12, 124, -12, 124, -12,
},
{
5, -2, 5, -2, 5, -2, 5, -2,
},
{
1, 0, 1, 0, 1, 0, 1, 0,
},
},
{
{
0, 0, 0, 0, 0, 0, 0, 0,
},
{
-1, 2, -1, 2, -1, 2, -1, 2,
},
{
-4, 8, -4, 8, -4, 8, -4, 8,
},
{
127, -6, 127, -6, 127, -6, 127, -6,
},
{
3, -1, 3, -1, 3, -1, 3, -1,
},
{
0, 0, 0, 0, 0, 0, 0, 0,
},
},
};
#endif
#endif
#if CONFIG_AOM_HIGHBITDEPTH
#if CONFIG_EXT_INTERP
DECLARE_ALIGNED(16, static const int16_t,
sub_pel_filters_12sharp_highbd_ver_signal_dir[15][6][8]) = {
{
{
0, 1, 0, 1, 0, 1, 0, 1,
},
{
-2, 3, -2, 3, -2, 3, -2, 3,
},
{
-7, 127, -7, 127, -7, 127, -7, 127,
},
{
8, -4, 8, -4, 8, -4, 8, -4,
},
{
2, -1, 2, -1, 2, -1, 2, -1,
},
{
1, 0, 1, 0, 1, 0, 1, 0,
},
},
{
{
-1, 2, -1, 2, -1, 2, -1, 2,
},
{
-3, 6, -3, 6, -3, 6, -3, 6,
},
{
-13, 124, -13, 124, -13, 124, -13, 124,
},
{
18, -8, 18, -8, 18, -8, 18, -8,
},
{