Commit 9bd42785 authored by Luc Trudeau's avatar Luc Trudeau

[CFL] SSSE3/AVX2 versions of luma_subsampling_420_lbd

Includes unit tests for conformance and speed.

SSSE2/SubsampleSpeedTest:
4x4: C time = 868 us, SIMD time = 200 us (~4.3x)
8x8: C time = 3054 us, SIMD time = 293 us (~10x)
16x16: C time = 11887 us, SIMD time = 760 us (~16x)

AVX2/SubsampleSpeedTest:
4x4: C time = 784 us, SIMD time = 205 us (~3.8x)
8x8: C time = 2774 us, SIMD time = 307 us (~9x)
16x16: C time = 10978 us, SIMD time = 489 us (~22x)

Change-Id: I7d5958097542599d57d1a9f9a0a1b809c6a345b0
parent 213ce98f
......@@ -401,6 +401,10 @@ if (CONFIG_CFL)
${AOM_AV1_COMMON_INTRIN_SSE2}
"${AOM_ROOT}/av1/common/cfl_sse2.c")
set(AOM_AV1_COMMON_INTRIN_SSSE3
${AOM_AV1_COMMON_INTRIN_SSSE3}
"${AOM_ROOT}/av1/common/cfl_ssse3.c")
set(AOM_AV1_COMMON_INTRIN_AVX2
${AOM_AV1_COMMON_INTRIN_AVX2}
"${AOM_ROOT}/av1/common/cfl_avx2.c")
......
......@@ -111,6 +111,7 @@ ifeq ($(CONFIG_CFL),yes)
AV1_COMMON_SRCS-yes += common/cfl.h
AV1_COMMON_SRCS-yes += common/cfl.c
AV1_COMMON_SRCS-$(HAVE_SSE2) += common/cfl_sse2.c
AV1_COMMON_SRCS-$(HAVE_SSSE3) += common/cfl_ssse3.c
AV1_COMMON_SRCS-$(HAVE_AVX2) += common/cfl_avx2.c
endif
......
......@@ -32,6 +32,10 @@ print <<EOF
#include "av1/common/daala_inv_txfm.h"
#endif
#if CONFIG_CFL
#include "av1/common/cfl.h"
#endif
struct macroblockd;
/* Encoder forward decls */
......@@ -582,6 +586,8 @@ if (aom_config("CONFIG_CFL") eq "yes") {
add_proto qw/void av1_cfl_subtract/, "int16_t *pred_buf_q3, int width, int height, int16_t avg_q3";
specialize qw/av1_cfl_subtract sse2 avx2/;
add_proto qw/cfl_subsample_lbd_fn get_subsample_lbd_fn/, "int sub_x, int sub_y";
specialize qw/get_subsample_lbd_fn ssse3 avx2/
}
1;
......@@ -254,7 +254,6 @@ void cfl_predict_block(MACROBLOCKD *const xd, uint8_t *dst, int dst_stride,
static void cfl_luma_subsampling_420_lbd(const uint8_t *input, int input_stride,
int16_t *output_q3, int width,
int height) {
assert((height - 1) * CFL_BUF_LINE + width <= CFL_BUF_SQUARE);
for (int j = 0; j < height; j++) {
for (int i = 0; i < width; i++) {
int top = i << 1;
......@@ -267,9 +266,8 @@ static void cfl_luma_subsampling_420_lbd(const uint8_t *input, int input_stride,
}
}
static void cfl_luma_subsampling_422_lbd(const uint8_t *input, int input_stride,
int16_t *output_q3, int width,
int height) {
void cfl_luma_subsampling_422_lbd(const uint8_t *input, int input_stride,
int16_t *output_q3, int width, int height) {
assert((height - 1) * CFL_BUF_LINE + width <= CFL_BUF_SQUARE);
for (int j = 0; j < height; j++) {
for (int i = 0; i < width; i++) {
......@@ -281,9 +279,8 @@ static void cfl_luma_subsampling_422_lbd(const uint8_t *input, int input_stride,
}
}
static void cfl_luma_subsampling_440_lbd(const uint8_t *input, int input_stride,
int16_t *output_q3, int width,
int height) {
void cfl_luma_subsampling_440_lbd(const uint8_t *input, int input_stride,
int16_t *output_q3, int width, int height) {
assert((height - 1) * CFL_BUF_LINE + width <= CFL_BUF_SQUARE);
for (int j = 0; j < height; j++) {
for (int i = 0; i < width; i++) {
......@@ -294,9 +291,8 @@ static void cfl_luma_subsampling_440_lbd(const uint8_t *input, int input_stride,
}
}
static void cfl_luma_subsampling_444_lbd(const uint8_t *input, int input_stride,
int16_t *output_q3, int width,
int height) {
void cfl_luma_subsampling_444_lbd(const uint8_t *input, int input_stride,
int16_t *output_q3, int width, int height) {
assert((height - 1) * CFL_BUF_LINE + width <= CFL_BUF_SQUARE);
for (int j = 0; j < height; j++) {
for (int i = 0; i < width; i++) {
......@@ -307,20 +303,21 @@ static void cfl_luma_subsampling_444_lbd(const uint8_t *input, int input_stride,
}
}
typedef void (*cfl_subsample_lbd_fn)(const uint8_t *input, int input_stride,
int16_t *output_q3, int width, int height);
static const cfl_subsample_lbd_fn subsample_lbd[2][2] = {
// (sub_y == 0, sub_x == 0) (sub_y == 0, sub_x == 1)
// (sub_y == 1, sub_x == 0) (sub_y == 1, sub_x == 1)
{ cfl_luma_subsampling_444_lbd, cfl_luma_subsampling_422_lbd },
{ cfl_luma_subsampling_440_lbd, cfl_luma_subsampling_420_lbd },
};
cfl_subsample_lbd_fn get_subsample_lbd_fn_c(int sub_x, int sub_y) {
static const cfl_subsample_lbd_fn subsample_lbd[2][2] = {
// (sub_y == 0, sub_x == 0) (sub_y == 0, sub_x == 1)
// (sub_y == 1, sub_x == 0) (sub_y == 1, sub_x == 1)
{ cfl_luma_subsampling_444_lbd, cfl_luma_subsampling_422_lbd },
{ cfl_luma_subsampling_440_lbd, cfl_luma_subsampling_420_lbd },
};
// AND sub_x and sub_y with 1 to ensures that an attacker won't be able to
// index the function pointer array out of bounds.
return subsample_lbd[sub_y & 1][sub_x & 1];
}
static void cfl_luma_subsampling_420_hbd(const uint16_t *input,
int input_stride, int16_t *output_q3,
int width, int height) {
assert((height - 1) * CFL_BUF_LINE + width <= CFL_BUF_SQUARE);
for (int j = 0; j < height; j++) {
for (int i = 0; i < width; i++) {
int top = i << 1;
......@@ -424,10 +421,8 @@ static void cfl_store(CFL_CTX *cfl, const uint8_t *input, int input_stride,
return;
}
(void)use_hbd;
// AND sub_x and sub_y with 1 to ensures that an attacker won't be able to
// index the function pointer array out of bounds.
subsample_lbd[sub_y & 1][sub_x & 1](input, input_stride, pred_buf_q3,
store_width, store_height);
get_subsample_lbd_fn(sub_x, sub_y)(input, input_stride, pred_buf_q3,
store_width, store_height);
}
// Adjust the row and column of blocks smaller than 8X8, as chroma-referenced
......
......@@ -14,6 +14,9 @@
#include "av1/common/blockd.h"
typedef void (*cfl_subsample_lbd_fn)(const uint8_t *input, int input_stride,
int16_t *output_q3, int width, int height);
static INLINE int is_cfl_allowed(const MB_MODE_INFO *mbmi) {
const BLOCK_SIZE bsize = mbmi->sb_type;
assert(bsize < BLOCK_SIZES_ALL);
......@@ -27,7 +30,7 @@ static INLINE int get_scaled_luma_q0(int alpha_q3, int16_t pred_buf_q3) {
static INLINE CFL_PRED_TYPE get_cfl_pred_type(PLANE_TYPE plane) {
assert(plane > 0);
return plane - 1;
return (CFL_PRED_TYPE)(plane - 1);
}
void cfl_predict_block(MACROBLOCKD *const xd, uint8_t *dst, int dst_stride,
......@@ -43,4 +46,14 @@ void cfl_store_dc_pred(MACROBLOCKD *const xd, const uint8_t *input,
void cfl_load_dc_pred(MACROBLOCKD *const xd, uint8_t *dst, int dst_stride,
TX_SIZE tx_size, CFL_PRED_TYPE pred_plane);
// TODO(ltrudeau) Remove this when 422 SIMD is added
void cfl_luma_subsampling_422_lbd(const uint8_t *input, int input_stride,
int16_t *output_q3, int width, int height);
// TODO(ltrudeau) Remove this when 440 SIMD is added
void cfl_luma_subsampling_440_lbd(const uint8_t *input, int input_stride,
int16_t *output_q3, int width, int height);
// TODO(ltrudeau) Remove this when 444 SIMD is added
void cfl_luma_subsampling_444_lbd(const uint8_t *input, int input_stride,
int16_t *output_q3, int width, int height);
#endif // AV1_COMMON_CFL_H_
......@@ -45,3 +45,57 @@ void av1_cfl_subtract_avx2(int16_t *pred_buf_q3, int width, int height,
_mm256_sub_epi16(val_x16, avg_x16));
} while ((pred_buf_q3 += stride) < end);
}
/**
* Adds 4 pixels (in a 2x2 grid) and multiplies them by 2. Resulting in a more
* precise version of a box filter 4:2:0 pixel subsampling in Q3.
*
* The CfL prediction buffer is always of size CFL_BUF_SQUARE. However, the
* active area is specified using width and height.
*
* Note: We don't need to worry about going over the active area, as long as we
* stay inside the CfL prediction buffer.
*
* Note: For 4:2:0 luma subsampling, the width will never be greater than 16.
*/
static void cfl_luma_subsampling_420_lbd_avx2(const uint8_t *input,
int input_stride,
int16_t *pred_buf_q3, int width,
int height) {
(void)width; // Max chroma width is 16, so all widths fit in one __m256i
const __m256i twos = _mm256_set1_epi8(2); // Thirty two twos
const int luma_stride = input_stride << 1;
const int16_t *end = pred_buf_q3 + height * CFL_BUF_LINE;
do {
// Load 32 values for the top and bottom rows.
// t_0, t_1, ... t_31
__m256i top = _mm256_loadu_si256((__m256i *)(input));
// b_0, b_1, ... b_31
__m256i bot = _mm256_loadu_si256((__m256i *)(input + input_stride));
// Horizontal add of the 32 values into 16 values that are multiplied by 2
// (t_0 + t_1) * 2, (t_2 + t_3) * 2, ... (t_30 + t_31) *2
top = _mm256_maddubs_epi16(top, twos);
// (b_0 + b_1) * 2, (b_2 + b_3) * 2, ... (b_30 + b_31) *2
bot = _mm256_maddubs_epi16(bot, twos);
// Add the 16 values in top with the 16 values in bottom
_mm256_storeu_si256((__m256i *)pred_buf_q3, _mm256_add_epi16(top, bot));
input += luma_stride;
pred_buf_q3 += CFL_BUF_LINE;
} while (pred_buf_q3 < end);
}
cfl_subsample_lbd_fn get_subsample_lbd_fn_avx2(int sub_x, int sub_y) {
static const cfl_subsample_lbd_fn subsample_lbd[2][2] = {
// (sub_y == 0, sub_x == 0) (sub_y == 0, sub_x == 1)
// (sub_y == 1, sub_x == 0) (sub_y == 1, sub_x == 1)
{ cfl_luma_subsampling_444_lbd, cfl_luma_subsampling_422_lbd },
{ cfl_luma_subsampling_440_lbd, cfl_luma_subsampling_420_lbd_avx2 },
};
// AND sub_x and sub_y with 1 to ensures that an attacker won't be able to
// index the function pointer array out of bounds.
return subsample_lbd[sub_y & 1][sub_x & 1];
}
/*
* Copyright (c) 2017, Alliance for Open Media. All rights reserved
*
* This source code is subject to the terms of the BSD 2 Clause License and
* the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
* was not distributed with this source code in the LICENSE file, you can
* obtain it at www.aomedia.org/license/software. If the Alliance for Open
* Media Patent License 1.0 was not distributed with this source code in the
* PATENTS file, you can obtain it at www.aomedia.org/license/patent.
*/
#include <tmmintrin.h>
#include "./av1_rtcd.h"
#include "av1/common/cfl.h"
/**
* Adds 4 pixels (in a 2x2 grid) and multiplies them by 2. Resulting in a more
* precise version of a box filter 4:2:0 pixel subsampling in Q3.
*
* The CfL prediction buffer is always of size CFL_BUF_SQUARE. However, the
* active area is specified using width and height.
*
* Note: We don't need to worry about going over the active area, as long as we
* stay inside the CfL prediction buffer.
*
* Note: For 4:2:0 luma subsampling, the width will never be greater than 16.
*/
static void cfl_luma_subsampling_420_lbd_ssse3(const uint8_t *input,
int input_stride,
int16_t *pred_buf_q3, int width,
int height) {
const __m128i twos = _mm_set1_epi8(2); // Sixteen twos
// Sixteen int8 values fit in one __m128i register. If this is enough to do
// the entire row, the next value is two rows down, otherwise we move to the
// next sixteen values.
const int next = (width == 16) ? 16 : input_stride << 1;
// Values in the prediction buffer are subsampled, so we only need to move
// down one row or forward by eight values.
const int next_chroma = (width == 16) ? 8 : CFL_BUF_LINE;
// When the width is less than 16, we double the stride, because we process
// four lines by iteration (instead of two).
const int luma_stride = input_stride << (1 + (width < 16));
const int chroma_stride = CFL_BUF_LINE << (width < 16);
const int16_t *end = pred_buf_q3 + height * CFL_BUF_LINE;
do {
// Load 16 values for the top and bottom rows.
// t_0, t_1, ... t_15
__m128i top = _mm_loadu_si128((__m128i *)(input));
// b_0, b_1, ... b_15
__m128i bot = _mm_loadu_si128((__m128i *)(input + input_stride));
// Load either the next line or the next 16 values
__m128i next_top = _mm_loadu_si128((__m128i *)(input + next));
__m128i next_bot =
_mm_loadu_si128((__m128i *)(input + next + input_stride));
// Horizontal add of the 16 values into 8 values that are multiplied by 2
// (t_0 + t_1) * 2, (t_2 + t_3) * 2, ... (t_14 + t_15) *2
top = _mm_maddubs_epi16(top, twos);
next_top = _mm_maddubs_epi16(next_top, twos);
// (b_0 + b_1) * 2, (b_2 + b_3) * 2, ... (b_14 + b_15) *2
bot = _mm_maddubs_epi16(bot, twos);
next_bot = _mm_maddubs_epi16(next_bot, twos);
// Add the 8 values in top with the 8 values in bottom
_mm_storeu_si128((__m128i *)pred_buf_q3, _mm_add_epi16(top, bot));
_mm_storeu_si128((__m128i *)(pred_buf_q3 + next_chroma),
_mm_add_epi16(next_top, next_bot));
input += luma_stride;
pred_buf_q3 += chroma_stride;
} while (pred_buf_q3 < end);
}
cfl_subsample_lbd_fn get_subsample_lbd_fn_ssse3(int sub_x, int sub_y) {
static const cfl_subsample_lbd_fn subsample_lbd[2][2] = {
// (sub_y == 0, sub_x == 0) (sub_y == 0, sub_x == 1)
// (sub_y == 1, sub_x == 0) (sub_y == 1, sub_x == 1)
{ cfl_luma_subsampling_444_lbd, cfl_luma_subsampling_422_lbd },
{ cfl_luma_subsampling_440_lbd, cfl_luma_subsampling_420_lbd_ssse3 },
};
// AND sub_x and sub_y with 1 to ensures that an attacker won't be able to
// index the function pointer array out of bounds.
return subsample_lbd[sub_y & 1][sub_x & 1];
}
......@@ -19,22 +19,32 @@ using std::tr1::make_tuple;
using libaom_test::ACMRandom;
#define NUM_ITERATIONS (100)
#define NUM_ITERATIONS (10)
#define NUM_ITERATIONS_SPEED (INT16_MAX)
#define ALL_SIZES_CFL(function) \
#define ALL_CFL_SIZES(function) \
make_tuple(4, 4, &function), make_tuple(8, 4, &function), \
make_tuple(4, 8, &function), make_tuple(8, 8, &function), \
make_tuple(16, 8, &function), make_tuple(8, 16, &function), \
make_tuple(16, 16, &function), make_tuple(32, 16, &function), \
make_tuple(16, 32, &function), make_tuple(32, 32, &function)
#define CHROMA_420_CFL_SIZES(function) \
make_tuple(4, 4, &function), make_tuple(8, 4, &function), \
make_tuple(4, 8, &function), make_tuple(8, 8, &function), \
make_tuple(16, 8, &function), make_tuple(8, 16, &function), \
make_tuple(16, 16, &function)
namespace {
typedef void (*subtract_fn)(int16_t *pred_buf_q3, int width, int height,
int16_t avg_q3);
typedef cfl_subsample_lbd_fn (*get_subsample_fn)(int width, int height);
typedef std::tr1::tuple<int, int, subtract_fn> subtract_param;
typedef std::tr1::tuple<int, int, get_subsample_fn> subsample_param;
static void assertFaster(int ref_elapsed_time, int elapsed_time) {
EXPECT_GT(ref_elapsed_time, elapsed_time)
<< "Error: CFLSubtractSpeedTest, SIMD slower than C." << std::endl
......@@ -58,11 +68,11 @@ class CFLSubtractTest : public ::testing::TestWithParam<subtract_param> {
virtual void SetUp() { subtract = GET_PARAM(2); }
protected:
int Width() const { return GET_PARAM(0); }
int Height() const { return GET_PARAM(1); }
int16_t pred_buf_data[CFL_BUF_SQUARE];
int16_t pred_buf_data_ref[CFL_BUF_SQUARE];
subtract_fn subtract;
int Width() const { return GET_PARAM(0); }
int Height() const { return GET_PARAM(1); }
void init(int width, int height) {
int k = 0;
for (int j = 0; j < height; j++) {
......@@ -74,6 +84,31 @@ class CFLSubtractTest : public ::testing::TestWithParam<subtract_param> {
}
};
class CFLSubsampleTest : public ::testing::TestWithParam<subsample_param> {
public:
virtual ~CFLSubsampleTest() {}
virtual void SetUp() { subsample = GET_PARAM(2); }
protected:
int Width() const { return GET_PARAM(0); }
int Height() const { return GET_PARAM(1); }
get_subsample_fn subsample;
uint8_t luma_pels[CFL_BUF_SQUARE];
uint8_t luma_pels_ref[CFL_BUF_SQUARE];
int16_t sub_luma_pels[CFL_BUF_SQUARE];
int16_t sub_luma_pels_ref[CFL_BUF_SQUARE];
void init(int width, int height) {
ACMRandom rnd(ACMRandom::DeterministicSeed());
for (int j = 0; j < height * 2; j++) {
for (int i = 0; i < width * 2; i++) {
const int val = rnd.Rand8();
luma_pels[j * CFL_BUF_LINE + i] = val;
luma_pels_ref[j * CFL_BUF_LINE + i] = val;
}
}
}
};
TEST_P(CFLSubtractTest, SubtractTest) {
const int width = Width();
const int height = Height();
......@@ -118,25 +153,83 @@ TEST_P(CFLSubtractTest, DISABLED_SubtractSpeedTest) {
aom_usec_timer_mark(&timer);
const int elapsed_time = (int)aom_usec_timer_elapsed(&timer);
#if 1
printSpeed(ref_elapsed_time, elapsed_time, width, height);
#endif
assertFaster(ref_elapsed_time, elapsed_time);
}
TEST_P(CFLSubsampleTest, SubsampleTest) {
const int width = Width();
const int height = Height();
for (int it = 0; it < NUM_ITERATIONS; it++) {
init(width, height);
subsample(1, 1)(luma_pels, CFL_BUF_LINE, sub_luma_pels, width, height);
get_subsample_lbd_fn_c(1, 1)(luma_pels_ref, CFL_BUF_LINE, sub_luma_pels_ref,
width, height);
for (int j = 0; j < height; j++) {
for (int i = 0; i < width; i++) {
ASSERT_EQ(sub_luma_pels_ref[j * CFL_BUF_LINE + i],
sub_luma_pels[j * CFL_BUF_LINE + i]);
}
}
}
}
TEST_P(CFLSubsampleTest, DISABLED_SubsampleSpeedTest) {
const int width = Width();
const int height = Height();
aom_usec_timer ref_timer;
aom_usec_timer timer;
init(width, height);
aom_usec_timer_start(&ref_timer);
for (int k = 0; k < NUM_ITERATIONS_SPEED; k++) {
get_subsample_lbd_fn_c(1, 1)(luma_pels, CFL_BUF_LINE, sub_luma_pels, width,
height);
}
aom_usec_timer_mark(&ref_timer);
int ref_elapsed_time = (int)aom_usec_timer_elapsed(&ref_timer);
aom_usec_timer_start(&timer);
for (int k = 0; k < NUM_ITERATIONS_SPEED; k++) {
subsample(1, 1)(luma_pels_ref, CFL_BUF_LINE, sub_luma_pels_ref, width,
height);
}
aom_usec_timer_mark(&timer);
int elapsed_time = (int)aom_usec_timer_elapsed(&timer);
printSpeed(ref_elapsed_time, elapsed_time, width, height);
assertFaster(ref_elapsed_time, elapsed_time);
}
#if HAVE_SSE2
const subtract_param subtract_sizes_sse2[] = { ALL_SIZES_CFL(
const subtract_param subtract_sizes_sse2[] = { ALL_CFL_SIZES(
av1_cfl_subtract_sse2) };
INSTANTIATE_TEST_CASE_P(SSE2, CFLSubtractTest,
::testing::ValuesIn(subtract_sizes_sse2));
#endif
#if HAVE_SSSE3
const subsample_param subsample_sizes_ssse3[] = { CHROMA_420_CFL_SIZES(
get_subsample_lbd_fn_ssse3) };
INSTANTIATE_TEST_CASE_P(SSSE3, CFLSubsampleTest,
::testing::ValuesIn(subsample_sizes_ssse3));
#endif
#if HAVE_AVX2
const subtract_param subtract_sizes_avx2[] = { ALL_SIZES_CFL(
const subtract_param subtract_sizes_avx2[] = { ALL_CFL_SIZES(
av1_cfl_subtract_avx2) };
const subsample_param subsample_sizes_avx2[] = { CHROMA_420_CFL_SIZES(
get_subsample_lbd_fn_avx2) };
INSTANTIATE_TEST_CASE_P(AVX2, CFLSubtractTest,
::testing::ValuesIn(subtract_sizes_avx2));
INSTANTIATE_TEST_CASE_P(AVX2, CFLSubsampleTest,
::testing::ValuesIn(subsample_sizes_avx2));
#endif
} // namespace
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment