Skip to content

GitLab

A
aom-rav1e
Commit a0d27597 authored by Angie Chiang's avatar Angie Chiang
Browse files
Options

Tune fwd txfm's config 

Maximize cos_bit's precision

Change-Id: Iad5d3915823f5c1c25a0caa3bd012d60caa2d521
parent 248f0557
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 60 additions and 60 deletions
av1/encoder/av1_fwd_txfm1d_cfg.h
View file @ a0d27597
This diff is collapsed.
av1/encoder/av1_fwd_txfm2d.c
View file @ a0d27597
......@@ -631,7 +631,7 @@ static const int8_t fwd_shift_16x32[3] = { 2, -4, 0 };
static const int8_t fwd_shift_32x16[3] = { 2, -4, 0 };
#if CONFIG_TX64X64
static const int8_t fwd_shift_32x64[3] = { 0, -2, -2 };
static const int8_t fwd_shift_64x32[3] = { 0, -2, -2 };
static const int8_t fwd_shift_64x32[3] = { 2, -4, -2 };
#endif
static const int8_t fwd_shift_4x16[3] = { 2, -1, 0 };
static const int8_t fwd_shift_16x4[3] = { 2, -1, 0 };
......@@ -639,7 +639,7 @@ static const int8_t fwd_shift_8x32[3] = { 2, -2, 0 };
static const int8_t fwd_shift_32x8[3] = { 2, -2, 0 };
#if CONFIG_TX64X64
static const int8_t fwd_shift_16x64[3] = { 0, -2, 0 };
static const int8_t fwd_shift_64x16[3] = { 0, -2, 0 };
static const int8_t fwd_shift_64x16[3] = { 2, -4, 0 };
#endif // CONFIG_TX64X64
const int8_t *fwd_txfm_shift_ls[TX_SIZES_ALL] = {
......
test/av1_fwd_txfm2d_test.cc
View file @ a0d27597
......@@ -139,33 +139,33 @@ vector<AV1FwdTxfm2dParam> GetTxfm2dParamList() {
for (int t = 0; t < TX_TYPES; ++t) {
const TX_TYPE tx_type = static_cast<TX_TYPE>(t);
param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_4X4, 2, 0.5));
param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_8X8, 5, 0.6));
param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_16X16, 11, 1.5));
param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_32X32, 70, 7));
param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_8X8, 5, 0.5));
param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_16X16, 11, 1.2));
param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_32X32, 70, 6.1));
#if CONFIG_TX64X64
if (tx_type == DCT_DCT) { // Other types not supported by these tx sizes.
param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_64X64, 70, 7));
param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_64X64, 64, 3.4));
}
#endif // CONFIG_TX64X64
param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_4X8, 3.2, 0.58));
param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_8X4, 3.6, 0.63));
param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_8X16, 15, 1.5));
param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_16X8, 15, 1.5));
param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_16X32, 57, 7));
param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_32X16, 37, 7));
param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_4X8, 3.2, 0.50));
param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_8X4, 3.2, 0.64));
param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_8X16, 8, 0.8));
param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_16X8, 8, 1.1));
param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_16X32, 29, 3.9));
param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_32X16, 37, 5.9));
param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_4X16, 5, 0.7));
param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_16X4, 5.5, 0.9));
param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_8X32, 20, 2.2));
param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_32X8, 15, 1.8));
param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_4X16, 3, 0.6));
param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_16X4, 5, 0.9));
param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_8X32, 21, 1.2));
param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_32X8, 13, 1.7));
#if CONFIG_TX64X64
if (tx_type == DCT_DCT) { // Other types not supported by these tx sizes.
param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_32X64, 136, 7));
param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_64X32, 136, 7));
param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_16X64, 30, 3.6));
param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_64X16, 50, 6.0));
param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_32X64, 136, 2.9));
param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_64X32, 136, 4.9));
param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_16X64, 30, 2.0));
param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_64X16, 36, 4.7));
}
#endif // CONFIG_TX64X64
}
......
test/av1_inv_txfm2d_test.cc
View file @ a0d27597
......@@ -155,7 +155,7 @@ vector<AV1InvTxfm2dParam> GetInvTxfm2dParamList() {
#endif // CONFIG_TX64X64
param_list.push_back(AV1InvTxfm2dParam(tx_type, TX_4X8, 2, 0.016));
param_list.push_back(AV1InvTxfm2dParam(tx_type, TX_8X4, 2, 0.025));
param_list.push_back(AV1InvTxfm2dParam(tx_type, TX_8X4, 2, 0.045));
param_list.push_back(AV1InvTxfm2dParam(tx_type, TX_8X16, 2, 0.2));
param_list.push_back(AV1InvTxfm2dParam(tx_type, TX_16X8, 2, 0.2));
param_list.push_back(AV1InvTxfm2dParam(tx_type, TX_16X32, 3, 0.4));
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment