Commit f1060f5b authored by Angie Chiang's avatar Angie Chiang
Browse files

Change dct32x32's range

Bitdepth 10/12:
Fit coefficient range into 32 bits
Fit codfficient * const range into 32 bits

Bitdepth 8:
Fit coefficient range into 16 bits
Fit codfficient * constant range into 32 bits

Change-Id: I50b5a3132e8a9f5155c971ab0f6eb52876d2b5ca
parent 39b3c025
......@@ -24,11 +24,7 @@ int get_tx_scale(const MACROBLOCKD *const xd, const TX_TYPE tx_type,
(void) tx_type;
#if CONFIG_VP9_HIGHBITDEPTH
if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) {
if (xd->bd == BITDEPTH_10) {
return 0;
} else {
return tx_size == TX_32X32;
}
return tx_size == TX_32X32;
}
#else
(void)xd;
......
......@@ -76,21 +76,21 @@ static const TXFM_2D_CFG fwd_txfm_2d_cfg_dct_dct_16 = {
TXFM_TYPE_DCT16}; // .txfm_type_row
// ---------------- config fwd_dct_dct_32 ----------------
static const int8_t fwd_shift_dct_dct_32[3] = {2, -3, 0};
static const int8_t fwd_shift_dct_dct_32[3] = {2, -4, 0};
static const int8_t fwd_stage_range_col_dct_dct_32[10] = {15, 16, 17, 18, 19,
20, 20, 20, 20, 20};
static const int8_t fwd_stage_range_row_dct_dct_32[10] = {17, 18, 19, 20, 21,
21, 21, 21, 21, 21};
static const int8_t fwd_stage_range_row_dct_dct_32[10] = {16, 17, 18, 19, 20,
20, 20, 20, 20, 20};
static const int8_t fwd_cos_bit_col_dct_dct_32[10] = {13, 13, 13, 13, 13,
12, 12, 12, 12, 12};
static const int8_t fwd_cos_bit_row_dct_dct_32[10] = {13, 13, 13, 12, 11,
11, 11, 11, 11, 11};
static const int8_t fwd_cos_bit_row_dct_dct_32[10] = {13, 13, 13, 13, 12,
12, 12, 12, 12, 12};
static const TXFM_2D_CFG fwd_txfm_2d_cfg_dct_dct_32 = {
32, // .txfm_size
10, // .stage_num_col
10, // .stage_num_row
// 0, // .log_scale
// 1, // .log_scale
fwd_shift_dct_dct_32, // .shift
fwd_stage_range_col_dct_dct_32, // .stage_range_col
fwd_stage_range_row_dct_dct_32, // .stage_range_row
......
......@@ -76,21 +76,21 @@ static const TXFM_2D_CFG inv_txfm_2d_cfg_dct_dct_16 = {
TXFM_TYPE_DCT16}; // .txfm_type_row
// ---------------- config inv_dct_dct_32 ----------------
static const int8_t inv_shift_dct_dct_32[2] = {-1, -6};
static const int8_t inv_stage_range_col_dct_dct_32[10] = {20, 20, 20, 20, 20,
20, 20, 20, 19, 19};
static const int8_t inv_stage_range_row_dct_dct_32[10] = {21, 21, 21, 21, 21,
21, 21, 21, 21, 21};
static const int8_t inv_cos_bit_col_dct_dct_32[10] = {12, 12, 12, 12, 12,
12, 12, 12, 12, 13};
static const int8_t inv_cos_bit_row_dct_dct_32[10] = {11, 11, 11, 11, 11,
11, 11, 11, 11, 11};
static const int8_t inv_shift_dct_dct_32[2] = {-1, -5};
static const int8_t inv_stage_range_col_dct_dct_32[10] = {19, 19, 19, 19, 19,
19, 19, 19, 18, 18};
static const int8_t inv_stage_range_row_dct_dct_32[10] = {20, 20, 20, 20, 20,
20, 20, 20, 20, 20};
static const int8_t inv_cos_bit_col_dct_dct_32[10] = {13, 13, 13, 13, 13,
13, 13, 13, 13, 13};
static const int8_t inv_cos_bit_row_dct_dct_32[10] = {12, 12, 12, 12, 12,
12, 12, 12, 12, 12};
static const TXFM_2D_CFG inv_txfm_2d_cfg_dct_dct_32 = {
32, // .txfm_size
10, // .stage_num_col
10, // .stage_num_row
// 0, // .log_scale
// 1, // .log_scale
inv_shift_dct_dct_32, // .shift
inv_stage_range_col_dct_dct_32, // .stage_range_col
inv_stage_range_row_dct_dct_32, // .stage_range_row
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment