Skip to content
GitLab
Commit a0d27597 authored by Angie Chiang's avatar Angie Chiang
Browse files

Tune fwd txfm's config 

Maximize cos_bit's precision

Change-Id: Iad5d3915823f5c1c25a0caa3bd012d60caa2d521
parent 248f0557
Hide whitespace changes
Inline Side-by-side
av1/encoder/av1_fwd_txfm1d_cfg.h
View file @ a0d27597
......@@ -109,15 +109,15 @@ static const int8_t fwd_stage_range_row_idx_64[1] = { 9 };
// cos bit
static const int8_t fwd_cos_bit_col_dct_64[12] = { 13, 13, 13, 13, 13, 13,
13, 13, 13, 13, 13, 13 };
static const int8_t fwd_cos_bit_row_dct_64[12] = { 13, 13, 12, 11, 10, 10,
static const int8_t fwd_cos_bit_row_dct_64[12] = { 10, 10, 10, 10, 10, 10,
10, 10, 10, 10, 10, 10 };
// ---------------- 4x8 1D constants -----------------------
static const int8_t fwd_stage_range_row_dct_4x8[4] = { 3, 4, 4, 4 };
static const int8_t fwd_stage_range_row_adst_4x8[6] = { 3, 3, 3, 4, 4, 4 };
static const int8_t fwd_stage_range_row_idx_4x8[1] = { 4 };
static const int8_t fwd_cos_bit_row_dct_4x8[6] = { 13, 12, 12, 12 };
static const int8_t fwd_cos_bit_row_adst_4x8[6] = { 13, 13, 12, 12, 12, 12 };
static const int8_t fwd_cos_bit_row_dct_4x8[6] = { 13, 13, 13, 13 };
static const int8_t fwd_cos_bit_row_adst_4x8[6] = { 13, 13, 13, 13, 13, 13 };
// ---------------- 8x4 1D constants -----------------------
static const int8_t fwd_stage_range_row_dct_8x4[6] = { 2, 3, 4, 4, 4, 4 };
......@@ -125,9 +125,9 @@ static const int8_t fwd_stage_range_row_adst_8x4[8] = {
2, 2, 2, 3, 3, 4, 4, 4
};
static const int8_t fwd_stage_range_row_idx_8x4[1] = { 3 };
static const int8_t fwd_cos_bit_row_dct_8x4[6] = { 13, 13, 12, 12, 12, 12 };
static const int8_t fwd_cos_bit_row_dct_8x4[6] = { 13, 13, 13, 13, 13, 13 };
static const int8_t fwd_cos_bit_row_adst_8x4[8] = { 13, 13, 13, 13,
12, 12, 12, 12 };
13, 13, 13, 13 };
// ---------------- 8x16 1D constants -----------------------
static const int8_t fwd_stage_range_row_dct_8x16[6] = { 4, 5, 6, 6, 6, 6 };
......@@ -135,9 +135,9 @@ static const int8_t fwd_stage_range_row_adst_8x16[8] = {
4, 4, 4, 5, 5, 6, 6, 6
};
static const int8_t fwd_stage_range_row_idx_8x16[1] = { 5 };
static const int8_t fwd_cos_bit_row_dct_8x16[6] = { 12, 12, 11, 11, 11, 11 };
static const int8_t fwd_cos_bit_row_adst_8x16[8] = { 12, 12, 12, 12,
11, 11, 11, 11 };
static const int8_t fwd_cos_bit_row_dct_8x16[6] = { 13, 13, 13, 13, 13, 13 };
static const int8_t fwd_cos_bit_row_adst_8x16[8] = { 13, 13, 13, 13,
13, 13, 13, 13 };
// ---------------- 16x8 1D constants -----------------------
static const int8_t fwd_stage_range_row_dct_16x8[8] = {
......@@ -146,10 +146,10 @@ static const int8_t fwd_stage_range_row_dct_16x8[8] = {
static const int8_t fwd_stage_range_row_adst_16x8[10] = { 3, 3, 3, 4, 4,
5, 5, 6, 6, 6 };
static const int8_t fwd_stage_range_row_idx_16x8[1] = { 5 };
static const int8_t fwd_cos_bit_row_dct_16x8[8] = { 12, 12, 12, 11,
11, 11, 11, 11 };
static const int8_t fwd_cos_bit_row_adst_16x8[10] = { 12, 12, 12, 12, 12,
12, 11, 11, 11, 11 };
static const int8_t fwd_cos_bit_row_dct_16x8[8] = { 13, 13, 13, 13,
13, 13, 13, 13 };
static const int8_t fwd_cos_bit_row_adst_16x8[10] = { 13, 13, 13, 13, 13,
13, 13, 13, 13, 13 };
// ---------------- 16x32 1D constants -----------------------
static const int8_t fwd_stage_range_row_dct_16x32[8] = {
......@@ -158,10 +158,10 @@ static const int8_t fwd_stage_range_row_dct_16x32[8] = {
static const int8_t fwd_stage_range_row_adst_16x32[10] = { 5, 5, 5, 6, 6,
7, 7, 8, 8, 8 };
static const int8_t fwd_stage_range_row_idx_16x32[1] = { 7 };
static const int8_t fwd_cos_bit_row_dct_16x32[8] = { 12, 12, 12, 11,
11, 11, 11, 11 };
static const int8_t fwd_cos_bit_row_adst_16x32[10] = { 12, 12, 12, 12, 12,
12, 11, 11, 11, 11 };
static const int8_t fwd_cos_bit_row_dct_16x32[8] = { 13, 13, 13, 13,
13, 13, 13, 13 };
static const int8_t fwd_cos_bit_row_adst_16x32[10] = { 13, 13, 13, 13, 13,
13, 13, 13, 13, 13 };
// ---------------- 32x16 1D constants -----------------------
static const int8_t fwd_stage_range_row_dct_32x16[10] = { 4, 5, 6, 7, 8,
......@@ -169,32 +169,32 @@ static const int8_t fwd_stage_range_row_dct_32x16[10] = { 4, 5, 6, 7, 8,
static const int8_t fwd_stage_range_row_adst_32x16[12] = { 4, 4, 4, 5, 5, 6,
6, 7, 7, 8, 8, 8 };
static const int8_t fwd_stage_range_row_idx_32x16[1] = { 6 };
static const int8_t fwd_cos_bit_row_dct_32x16[10] = { 12, 12, 12, 12, 11,
11, 11, 11, 11, 11 };
static const int8_t fwd_cos_bit_row_adst_32x16[12] = { 12, 12, 12, 12, 12, 12,
12, 12, 11, 11, 11, 11 };
static const int8_t fwd_cos_bit_row_dct_32x16[10] = { 13, 13, 13, 13, 13,
13, 13, 13, 13, 13 };
static const int8_t fwd_cos_bit_row_adst_32x16[12] = { 13, 13, 13, 13, 13, 13,
13, 13, 13, 13, 13, 13 };
// ---------------- 32x64 1D constants -----------------------
static const int8_t fwd_stage_range_row_dct_32x64[10] = { 6, 7, 8, 9, 10,
10, 10, 10, 10, 10 };
static const int8_t fwd_stage_range_row_idx_32x64[1] = { 8 };
static const int8_t fwd_cos_bit_row_dct_32x64[10] = { 13, 12, 12, 11, 10,
10, 10, 10, 10, 10 };
static const int8_t fwd_cos_bit_row_dct_32x64[10] = { 11, 11, 11, 11, 11,
11, 11, 11, 11, 11 };
// ---------------- 64x32 1D constants -----------------------
static const int8_t fwd_stage_range_row_dct_64x32[12] = {
5, 6, 7, 8, 9, 10, 10, 10, 10, 10, 10, 10
};
static const int8_t fwd_stage_range_row_idx_64x32[1] = { 8 };
static const int8_t fwd_cos_bit_row_dct_64x32[12] = { 13, 13, 12, 11, 10, 10,
10, 10, 10, 10, 10, 10 };
static const int8_t fwd_cos_bit_row_dct_64x32[12] = { 11, 11, 11, 11, 11, 11,
11, 11, 11, 11, 11, 11 };
// ---------------- 4x16 1D constants -----------------------
static const int8_t fwd_stage_range_row_dct_4x16[4] = { 4, 5, 5, 5 };
static const int8_t fwd_stage_range_row_adst_4x16[6] = { 4, 4, 4, 5, 5, 5 };
static const int8_t fwd_stage_range_row_idx_4x16[1] = { 5 };
static const int8_t fwd_cos_bit_row_dct_4x16[6] = { 12, 12, 12, 12 };
static const int8_t fwd_cos_bit_row_adst_4x16[6] = { 12, 12, 12, 12, 12, 12 };
static const int8_t fwd_cos_bit_row_dct_4x16[6] = { 13, 13, 13, 13 };
static const int8_t fwd_cos_bit_row_adst_4x16[6] = { 13, 13, 13, 13, 13, 13 };
// ---------------- 16x4 1D constants -----------------------
static const int8_t fwd_stage_range_row_dct_16x4[8] = {
......@@ -203,10 +203,10 @@ static const int8_t fwd_stage_range_row_dct_16x4[8] = {
static const int8_t fwd_stage_range_row_adst_16x4[10] = { 2, 2, 2, 3, 3,
4, 4, 5, 5, 5 };
static const int8_t fwd_stage_range_row_idx_16x4[1] = { 4 };
static const int8_t fwd_cos_bit_row_dct_16x4[8] = { 12, 12, 12, 12,
12, 12, 12, 12 };
static const int8_t fwd_cos_bit_row_adst_16x4[10] = { 12, 12, 12, 12, 12,
12, 12, 12, 12, 12 };
static const int8_t fwd_cos_bit_row_dct_16x4[8] = { 13, 13, 13, 13,
13, 13, 13, 13 };
static const int8_t fwd_cos_bit_row_adst_16x4[10] = { 13, 13, 13, 13, 13,
13, 13, 13, 13, 13 };
// ---------------- 8x32 1D constants -----------------------
static const int8_t fwd_stage_range_row_dct_8x32[6] = { 5, 6, 7, 7, 7, 7 };
......@@ -214,9 +214,9 @@ static const int8_t fwd_stage_range_row_adst_8x32[8] = {
5, 5, 5, 6, 6, 7, 7, 7
};
static const int8_t fwd_stage_range_row_idx_8x32[1] = { 6 };
static const int8_t fwd_cos_bit_row_dct_8x32[6] = { 12, 12, 11, 11, 11, 11 };
static const int8_t fwd_cos_bit_row_dct_8x32[6] = { 12, 12, 12, 12, 12, 12 };
static const int8_t fwd_cos_bit_row_adst_8x32[8] = { 12, 12, 12, 12,
11, 11, 11, 11 };
12, 12, 12, 12 };
// ---------------- 32x8 1D constants -----------------------
static const int8_t fwd_stage_range_row_dct_32x8[10] = { 3, 4, 5, 6, 7,
......@@ -224,25 +224,25 @@ static const int8_t fwd_stage_range_row_dct_32x8[10] = { 3, 4, 5, 6, 7,
static const int8_t fwd_stage_range_row_adst_32x8[12] = { 3, 3, 3, 4, 4, 5,
5, 6, 6, 7, 7, 7 };
static const int8_t fwd_stage_range_row_idx_32x8[1] = { 5 };
static const int8_t fwd_cos_bit_row_dct_32x8[10] = { 12, 12, 12, 12, 11,
11, 11, 11, 11, 11 };
static const int8_t fwd_cos_bit_row_dct_32x8[10] = { 12, 12, 12, 12, 12,
12, 12, 12, 12, 12 };
static const int8_t fwd_cos_bit_row_adst_32x8[12] = { 12, 12, 12, 12, 12, 12,
12, 11, 11, 11, 11, 11 };
12, 12, 12, 12, 12, 12 };
// ---------------- 16x64 1D constants -----------------------
static const int8_t fwd_stage_range_row_dct_16x64[8] = {
6, 7, 8, 9, 9, 9, 9, 9
};
static const int8_t fwd_stage_range_row_idx_16x64[1] = { 8 };
static const int8_t fwd_cos_bit_row_dct_16x64[8] = { 12, 11, 10, 10,
10, 10, 10, 10 };
static const int8_t fwd_cos_bit_row_dct_16x64[8] = { 12, 12, 12, 12,
12, 12, 12, 12 };
// ---------------- 64x16 1D constants -----------------------
static const int8_t fwd_stage_range_row_dct_64x16[12] = { 4, 5, 6, 7, 8, 9,
9, 9, 9, 9, 9, 9 };
static const int8_t fwd_stage_range_row_idx_64x16[1] = { 7 };
static const int8_t fwd_cos_bit_row_dct_64x16[12] = { 13, 13, 12, 11, 11, 11,
11, 11, 11, 11, 11, 11 };
static const int8_t fwd_cos_bit_row_dct_64x16[12] = { 12, 12, 12, 12, 12, 12,
12, 12, 12, 12, 12, 12 };
//
// ---------------- row config fwd_dct_4 ----------------
......
av1/encoder/av1_fwd_txfm2d.c
View file @ a0d27597
......@@ -631,7 +631,7 @@ static const int8_t fwd_shift_16x32[3] = { 2, -4, 0 };
static const int8_t fwd_shift_32x16[3] = { 2, -4, 0 };
#if CONFIG_TX64X64
static const int8_t fwd_shift_32x64[3] = { 0, -2, -2 };
static const int8_t fwd_shift_64x32[3] = { 0, -2, -2 };
static const int8_t fwd_shift_64x32[3] = { 2, -4, -2 };
#endif
static const int8_t fwd_shift_4x16[3] = { 2, -1, 0 };
static const int8_t fwd_shift_16x4[3] = { 2, -1, 0 };
......@@ -639,7 +639,7 @@ static const int8_t fwd_shift_8x32[3] = { 2, -2, 0 };
static const int8_t fwd_shift_32x8[3] = { 2, -2, 0 };
#if CONFIG_TX64X64
static const int8_t fwd_shift_16x64[3] = { 0, -2, 0 };
static const int8_t fwd_shift_64x16[3] = { 0, -2, 0 };
static const int8_t fwd_shift_64x16[3] = { 2, -4, 0 };
#endif // CONFIG_TX64X64
const int8_t *fwd_txfm_shift_ls[TX_SIZES_ALL] = {
......
test/av1_fwd_txfm2d_test.cc
View file @ a0d27597
......@@ -139,33 +139,33 @@ vector<AV1FwdTxfm2dParam> GetTxfm2dParamList() {
for (int t = 0; t < TX_TYPES; ++t) {
const TX_TYPE tx_type = static_cast<TX_TYPE>(t);
param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_4X4, 2, 0.5));
param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_8X8, 5, 0.6));
param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_16X16, 11, 1.5));
param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_32X32, 70, 7));
param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_8X8, 5, 0.5));
param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_16X16, 11, 1.2));
param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_32X32, 70, 6.1));
#if CONFIG_TX64X64
if (tx_type == DCT_DCT) { // Other types not supported by these tx sizes.
param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_64X64, 70, 7));
param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_64X64, 64, 3.4));
}
#endif // CONFIG_TX64X64
param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_4X8, 3.2, 0.58));
param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_8X4, 3.6, 0.63));
param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_8X16, 15, 1.5));
param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_16X8, 15, 1.5));
param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_16X32, 57, 7));
param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_32X16, 37, 7));
param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_4X8, 3.2, 0.50));
param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_8X4, 3.2, 0.64));
param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_8X16, 8, 0.8));
param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_16X8, 8, 1.1));
param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_16X32, 29, 3.9));
param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_32X16, 37, 5.9));
param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_4X16, 5, 0.7));
param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_16X4, 5.5, 0.9));
param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_8X32, 20, 2.2));
param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_32X8, 15, 1.8));
param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_4X16, 3, 0.6));
param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_16X4, 5, 0.9));
param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_8X32, 21, 1.2));
param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_32X8, 13, 1.7));
#if CONFIG_TX64X64
if (tx_type == DCT_DCT) { // Other types not supported by these tx sizes.
param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_32X64, 136, 7));
param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_64X32, 136, 7));
param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_16X64, 30, 3.6));
param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_64X16, 50, 6.0));
param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_32X64, 136, 2.9));
param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_64X32, 136, 4.9));
param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_16X64, 30, 2.0));
param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_64X16, 36, 4.7));
}
#endif // CONFIG_TX64X64
}
......
test/av1_inv_txfm2d_test.cc
View file @ a0d27597
......@@ -155,7 +155,7 @@ vector<AV1InvTxfm2dParam> GetInvTxfm2dParamList() {
#endif // CONFIG_TX64X64
param_list.push_back(AV1InvTxfm2dParam(tx_type, TX_4X8, 2, 0.016));
param_list.push_back(AV1InvTxfm2dParam(tx_type, TX_8X4, 2, 0.025));
param_list.push_back(AV1InvTxfm2dParam(tx_type, TX_8X4, 2, 0.045));
param_list.push_back(AV1InvTxfm2dParam(tx_type, TX_8X16, 2, 0.2));
param_list.push_back(AV1InvTxfm2dParam(tx_type, TX_16X8, 2, 0.2));
param_list.push_back(AV1InvTxfm2dParam(tx_type, TX_16X32, 3, 0.4));
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment