Commit 1158bff7 authored by Debargha Mukherjee's avatar Debargha Mukherjee

Various fixes to scale managed txfms and tests

This patch clears all test failures with coeff range checking
enabled for forward and inverse transforms. Also this
patch ensures that there are no transposes for any of the
rectangular transforms.
Some fine-tunnig and refactoring are still pending.
Some of the tests still need to be rewritten.

Change-Id: Ib0e3a4ceccef665ba007d121f536fad7135f38d5
parent f0930dca
......@@ -16,8 +16,13 @@
// sum of fwd_shift_##
#if CONFIG_TX64X64
static const int8_t fwd_shift_sum[TX_SIZES] = { 2, 1, 0, -2, -4 };
#else // CONFIG_TX64X64
static const int8_t inv_start_range[TX_SIZES_ALL] = {
5, 6, 7, 7, 7, 6, 6, 7, 7, 7, 7, 7, 7, 6, 6, 7, 7, 7, 7
};
#else // CONFIG_TX64X64
static const int8_t fwd_shift_sum[TX_SIZES] = { 2, 1, 0, -2 };
static const int8_t inv_start_range[TX_SIZES_ALL] = { 5, 6, 7, 7, 6, 6, 7,
7, 7, 7, 6, 6, 7, 7 };
#endif // CONFIG_TX64X64
// ---------------- 4x4 1D config -----------------------
......@@ -25,7 +30,7 @@ static const int8_t fwd_shift_sum[TX_SIZES] = { 2, 1, 0, -2 };
static const int8_t inv_shift_4[2] = { 0, -4 };
// stage range
static const int8_t inv_stage_range_col_dct_4[4] = { 3, 3, 2, 2 };
static const int8_t inv_stage_range_col_dct_4[4] = { 3, 3, 3, 3 };
static const int8_t inv_stage_range_row_dct_4[4] = { 3, 3, 3, 3 };
static const int8_t inv_stage_range_col_adst_4[6] = { 3, 3, 3, 3, 2, 2 };
static const int8_t inv_stage_range_row_adst_4[6] = { 3, 3, 3, 3, 3, 3 };
......@@ -111,11 +116,18 @@ static const int8_t inv_cos_bit_row_adst_16[10] = { 12, 12, 12, 12, 12,
// ---------------- 8x16 1D constants -----------------------
#define inv_shift_8x16 inv_shift_16
// stage range
static const int8_t inv_stage_range_row_dct_8x16[6] = { 5, 5, 5, 5, 5, 5 };
static const int8_t inv_stage_range_row_adst_8x16[8] = {
5, 5, 5, 5, 5, 5, 5, 5
};
static const int8_t inv_stage_range_col_dct_8x16[8] =
ARRAYOFFSET8(-2, 7, 7, 7, 7, 7, 7, 7, 7);
static const int8_t inv_stage_range_col_adst_8x16[10] =
ARRAYOFFSET10(-2, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7);
// cos bit
static const int8_t inv_cos_bit_row_dct_8x16[6] = { 12, 12, 12, 12, 12, 12 };
static const int8_t inv_cos_bit_row_adst_8x16[8] = { 12, 12, 12, 12,
12, 12, 12, 12 };
static const int8_t inv_cos_bit_col_dct_8x16[8] = { 13, 13, 13, 13,
13, 13, 13, 13 };
static const int8_t inv_cos_bit_col_adst_8x16[10] = { 13, 13, 13, 13, 13,
......@@ -249,11 +261,18 @@ static const int8_t inv_cos_bit_col_adst_16x4[6] = { 13, 13, 13, 13, 13, 13 };
// ---------------- 8x32 1D constants -----------------------
#define inv_shift_8x32 inv_shift_32
// stage range
static const int8_t inv_stage_range_row_dct_8x32[6] = { 5, 5, 5, 5, 5, 5 };
static const int8_t inv_stage_range_row_adst_8x32[8] = {
5, 5, 5, 5, 5, 5, 5, 5
};
static const int8_t inv_stage_range_col_dct_8x32[10] =
ARRAYOFFSET10(-4, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9);
static const int8_t inv_stage_range_col_adst_8x32[12] =
ARRAYOFFSET12(-4, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9);
// cos bit
static const int8_t inv_cos_bit_row_dct_8x32[6] = { 12, 12, 12, 12, 12, 12 };
static const int8_t inv_cos_bit_row_adst_8x32[8] = { 12, 12, 12, 12,
12, 12, 12, 12 };
static const int8_t inv_cos_bit_col_dct_8x32[10] = { 13, 13, 13, 13, 13,
13, 13, 13, 13, 13 };
static const int8_t inv_cos_bit_col_adst_8x32[12] = { 13, 13, 13, 13, 13, 13,
......@@ -692,6 +711,16 @@ static const TXFM_1D_CFG inv_txfm_1d_col_cfg_adst_4x16 = {
TXFM_TYPE_ADST16, // .txfm_type
};
// ---------------- row config inv_dct_8x16 ----------------
static const TXFM_1D_CFG inv_txfm_1d_row_cfg_dct_8x16 = {
8, // .txfm_size
6, // .stage_num
inv_shift_8x16, // .shift
inv_stage_range_row_dct_8x16, // .stage_range
inv_cos_bit_row_dct_8x16, // .cos_bit
TXFM_TYPE_DCT8 // .txfm_type
};
// ---------------- col config inv_dct_8x16 ----------------
static const TXFM_1D_CFG inv_txfm_1d_col_cfg_dct_8x16 = {
16, // .txfm_size
......@@ -702,6 +731,16 @@ static const TXFM_1D_CFG inv_txfm_1d_col_cfg_dct_8x16 = {
TXFM_TYPE_DCT16 // .txfm_type
};
// ---------------- row config inv_adst_8x16 ----------------
static const TXFM_1D_CFG inv_txfm_1d_row_cfg_adst_8x16 = {
8, // .txfm_size
8, // .stage_num
inv_shift_8x16, // .shift
inv_stage_range_row_adst_8x16, // .stage_range
inv_cos_bit_row_adst_8x16, // .cos_bit
TXFM_TYPE_ADST8, // .txfm_type
};
// ---------------- col config inv_adst_8x16 ----------------
static const TXFM_1D_CFG inv_txfm_1d_col_cfg_adst_8x16 = {
16, // .txfm_size
......@@ -744,6 +783,16 @@ static const TXFM_1D_CFG inv_txfm_1d_col_cfg_dct_64x16 = {
};
#endif // CONFIG_TX64X64
// ---------------- row config inv_dct_8x32 ----------------
static const TXFM_1D_CFG inv_txfm_1d_row_cfg_dct_8x32 = {
8, // .txfm_size
6, // .stage_num
inv_shift_8x32, // .shift
inv_stage_range_row_dct_8x32, // .stage_range
inv_cos_bit_row_dct_8x32, // .cos_bit_col
TXFM_TYPE_DCT8 // .txfm_type
};
// ---------------- col config inv_dct_8x32 ----------------
static const TXFM_1D_CFG inv_txfm_1d_col_cfg_dct_8x32 = {
32, // .txfm_size
......@@ -754,6 +803,16 @@ static const TXFM_1D_CFG inv_txfm_1d_col_cfg_dct_8x32 = {
TXFM_TYPE_DCT32 // .txfm_type
};
// ---------------- row config inv_adst_8x32 ----------------
static const TXFM_1D_CFG inv_txfm_1d_row_cfg_adst_8x32 = {
8, // .txfm_size
8, // .stage_num
inv_shift_8x32, // .shift
inv_stage_range_row_adst_8x32, // .stage_range
inv_cos_bit_row_adst_8x32, // .cos_bit
TXFM_TYPE_ADST8, // .txfm_type
};
// ---------------- col config inv_adst_8x32 ----------------
static const TXFM_1D_CFG inv_txfm_1d_col_cfg_adst_8x32 = {
32, // .txfm_size
......
This diff is collapsed.
......@@ -263,7 +263,7 @@ void av1_gen_fwd_stage_range(int8_t *stage_range_col, int8_t *stage_range_row,
const TXFM_2D_FLIP_CFG *cfg, int bd);
void av1_gen_inv_stage_range(int8_t *stage_range_col, int8_t *stage_range_row,
const TXFM_2D_FLIP_CFG *cfg, int8_t fwd_shift,
const TXFM_2D_FLIP_CFG *cfg, TX_SIZE tx_size,
int bd);
void av1_get_fwd_txfm_cfg(TX_TYPE tx_type, TX_SIZE tx_size,
......
......@@ -209,8 +209,8 @@ static const int8_t fwd_stage_range_row_dct_4x16[4] =
ARRAYOFFSET4(4, 0, 1, 2, 2);
static const int8_t fwd_stage_range_row_adst_4x16[6] =
ARRAYOFFSET6(4, 0, 0, 1, 2, 2, 2);
static const int8_t fwd_cos_bit_row_dct_4x16[6] = { 13, 13, 13, 13 };
static const int8_t fwd_cos_bit_row_adst_4x16[6] = { 13, 13, 13, 13, 13, 13 };
static const int8_t fwd_cos_bit_row_dct_4x16[6] = { 12, 12, 12, 12 };
static const int8_t fwd_cos_bit_row_adst_4x16[6] = { 12, 12, 12, 12, 12, 12 };
// ---------------- 16x4 1D constants -----------------------
#define fwd_shift_16x4 fwd_shift_16
......@@ -218,10 +218,10 @@ static const int8_t fwd_stage_range_row_dct_16x4[8] =
ARRAYOFFSET8(2, 0, 1, 2, 3, 4, 4, 4, 4);
static const int8_t fwd_stage_range_row_adst_16x4[10] =
ARRAYOFFSET10(2, 0, 0, 1, 2, 2, 3, 3, 4, 4, 4);
static const int8_t fwd_cos_bit_row_dct_16x4[8] = { 13, 13, 13, 13,
13, 13, 13, 13 };
static const int8_t fwd_cos_bit_row_adst_16x4[10] = { 13, 13, 13, 13, 13,
13, 13, 13, 13, 13 };
static const int8_t fwd_cos_bit_row_dct_16x4[8] = { 12, 12, 12, 12,
12, 12, 12, 12 };
static const int8_t fwd_cos_bit_row_adst_16x4[10] = { 12, 12, 12, 12, 12,
12, 12, 12, 12, 12 };
// ---------------- 8x32 1D constants -----------------------
#define fwd_shift_8x32 fwd_shift_32
......@@ -229,9 +229,9 @@ static const int8_t fwd_stage_range_row_dct_8x32[6] =
ARRAYOFFSET6(5, 0, 1, 2, 3, 3, 3);
static const int8_t fwd_stage_range_row_adst_8x32[8] =
ARRAYOFFSET8(5, 0, 0, 1, 2, 2, 3, 3, 3);
static const int8_t fwd_cos_bit_row_dct_8x32[6] = { 13, 13, 13, 13, 13, 13 };
static const int8_t fwd_cos_bit_row_adst_8x32[8] = { 13, 13, 13, 13,
13, 13, 13, 13 };
static const int8_t fwd_cos_bit_row_dct_8x32[6] = { 12, 12, 11, 11, 11, 11 };
static const int8_t fwd_cos_bit_row_adst_8x32[8] = { 12, 12, 12, 12,
11, 11, 11, 11 };
// ---------------- 32x8 1D constants -----------------------
#define fwd_shift_32x8 fwd_shift_32
......@@ -239,17 +239,17 @@ static const int8_t fwd_stage_range_row_dct_32x8[10] =
ARRAYOFFSET10(3, 0, 1, 2, 3, 4, 5, 5, 5, 5, 5);
static const int8_t fwd_stage_range_row_adst_32x8[12] =
ARRAYOFFSET12(3, 0, 0, 1, 2, 2, 3, 3, 4, 4, 5, 5, 5);
static const int8_t fwd_cos_bit_row_dct_32x8[10] = { 12, 12, 12, 12, 12,
12, 12, 12, 12, 12 };
static const int8_t fwd_cos_bit_row_dct_32x8[10] = { 12, 12, 12, 12, 11,
11, 11, 11, 11, 11 };
static const int8_t fwd_cos_bit_row_adst_32x8[12] = { 12, 12, 12, 12, 12, 12,
12, 12, 12, 12, 12, 12 };
12, 11, 11, 11, 11, 11 };
// ---------------- 16x64 1D constants -----------------------
#define fwd_shift_16x64 fwd_shift_64
static const int8_t fwd_stage_range_row_dct_16x64[8] =
ARRAYOFFSET8(6, 0, 1, 2, 3, 4, 4, 4, 4);
static const int8_t fwd_cos_bit_row_dct_16x64[8] = { 12, 12, 12, 11,
11, 11, 11, 11 };
static const int8_t fwd_cos_bit_row_dct_16x64[8] = { 12, 11, 10, 10,
10, 10, 10, 10 };
// ---------------- 64x16 1D constants -----------------------
#define fwd_shift_64x16 fwd_shift_64
......
This diff is collapsed.
......@@ -148,24 +148,24 @@ vector<AV1FwdTxfm2dParam> GetTxfm2dParamList() {
}
#endif // CONFIG_TX64X64
param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_4X8, 2.9, 0.55));
param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_8X4, 3.2, 0.56));
param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_4X8, 3.2, 0.58));
param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_8X4, 3.2, 0.58));
param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_8X16, 6.5, 1));
param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_16X8, 6.5, 1));
param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_16X32, 46, 7));
param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_16X32, 50, 7));
param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_32X16, 30, 7));
param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_4X16, 5, 0.7));
param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_16X4, 5.5, 0.9));
param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_8X32, 14, 2.1));
param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_32X8, 11, 1.6));
param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_8X32, 20, 2.2));
param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_32X8, 15, 1.8));
#if CONFIG_TX64X64
if (tx_type == DCT_DCT) { // Other types not supported by these tx sizes.
param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_32X64, 136, 7));
param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_64X32, 136, 7));
param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_16X64, 16, 1.6));
param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_64X16, 20, 2.0));
param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_16X64, 30, 3.6));
param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_64X16, 50, 6.0));
}
#endif // CONFIG_TX64X64
}
......
......@@ -158,20 +158,20 @@ vector<AV1InvTxfm2dParam> GetInvTxfm2dParamList() {
param_list.push_back(AV1InvTxfm2dParam(tx_type, TX_8X4, 2, 0.016));
param_list.push_back(AV1InvTxfm2dParam(tx_type, TX_8X16, 2, 0.033));
param_list.push_back(AV1InvTxfm2dParam(tx_type, TX_16X8, 2, 0.033));
param_list.push_back(AV1InvTxfm2dParam(tx_type, TX_16X32, 2, 0.4));
param_list.push_back(AV1InvTxfm2dParam(tx_type, TX_32X16, 2, 0.4));
param_list.push_back(AV1InvTxfm2dParam(tx_type, TX_16X32, 3, 0.4));
param_list.push_back(AV1InvTxfm2dParam(tx_type, TX_32X16, 3, 0.4));
param_list.push_back(AV1InvTxfm2dParam(tx_type, TX_4X16, 2, 0.1));
param_list.push_back(AV1InvTxfm2dParam(tx_type, TX_16X4, 2, 0.1));
param_list.push_back(AV1InvTxfm2dParam(tx_type, TX_4X16, 2, 0.2));
param_list.push_back(AV1InvTxfm2dParam(tx_type, TX_16X4, 2, 0.2));
param_list.push_back(AV1InvTxfm2dParam(tx_type, TX_8X32, 2, 0.2));
param_list.push_back(AV1InvTxfm2dParam(tx_type, TX_32X8, 2, 0.1));
param_list.push_back(AV1InvTxfm2dParam(tx_type, TX_32X8, 2, 0.2));
#if CONFIG_TX64X64
if (tx_type == DCT_DCT) { // Other types not supported by these tx sizes.
param_list.push_back(AV1InvTxfm2dParam(tx_type, TX_32X64, 3, 0.3));
param_list.push_back(AV1InvTxfm2dParam(tx_type, TX_64X32, 3, 0.31));
param_list.push_back(AV1InvTxfm2dParam(tx_type, TX_16X64, 2, 0.16));
param_list.push_back(AV1InvTxfm2dParam(tx_type, TX_64X16, 2, 0.16));
param_list.push_back(AV1InvTxfm2dParam(tx_type, TX_32X64, 4, 0.38));
param_list.push_back(AV1InvTxfm2dParam(tx_type, TX_64X32, 4, 0.38));
param_list.push_back(AV1InvTxfm2dParam(tx_type, TX_16X64, 3, 0.38));
param_list.push_back(AV1InvTxfm2dParam(tx_type, TX_64X16, 3, 0.38));
}
#endif // CONFIG_TX64X64
}
......@@ -201,9 +201,8 @@ TEST(AV1InvTxfm2d, CfgTest) {
static_cast<TX_SIZE>(tx_size), &cfg);
int8_t stage_range_col[MAX_TXFM_STAGE_NUM];
int8_t stage_range_row[MAX_TXFM_STAGE_NUM];
const TX_SIZE tx_size_sqr_up = txsize_sqr_up_map[tx_size];
av1_gen_inv_stage_range(stage_range_col, stage_range_row, &cfg,
fwd_shift_sum[tx_size_sqr_up], bd);
(TX_SIZE)tx_size, bd);
const TXFM_1D_CFG *col_cfg = cfg.col_cfg;
const TXFM_1D_CFG *row_cfg = cfg.row_cfg;
libaom_test::txfm_stage_range_check(stage_range_col, col_cfg->stage_num,
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment