Commit a30b9ec6 authored by Urvang Joshi's avatar Urvang Joshi Committed by Debargha Mukherjee

Correct scale factor for TX_16X64 and TX_64X16.

BUG=aomedia:1114

Change-Id: I7fbeb4c2da996801b945304e182403ec325f95bc
parent 15b0113b
......@@ -84,10 +84,9 @@ static INLINE void fwd_txfm2d_c(const int16_t *input, int32_t *output,
int rect_type2_shift = 0;
if (rect_type == 2 || rect_type == -2) {
const int txfm_size_max = AOMMAX(txfm_size_col, txfm_size_row);
// For 64x16 / 16x64 shift 3 bits, for 32x8 / 8x32 shift 2 bits, for
// 16x4 / 4x16 shift by 1 bit.
rect_type2_shift =
(txfm_size_max == 64 ? 3 : (txfm_size_max == 32 ? 2 : 1));
// For 64x16 / 16x64 / 32x8 / 8x32 shift 2 bits, and
// For 16x4 / 4x16 shift by 1 bit.
rect_type2_shift = (txfm_size_max >= 32) ? 2 : 1;
}
// Take the shift from the larger dimension in the rectangular case.
const int8_t *shift = (txfm_size_col > txfm_size_row) ? cfg->row_cfg->shift
......
......@@ -162,7 +162,7 @@ vector<AV1FwdTxfm2dParam> GetTxfm2dParamList() {
param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_32X64, 136, 7));
param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_64X32, 136, 7));
param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_16X64, 16, 1.6));
param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_64X16, 16, 1.6));
param_list.push_back(AV1FwdTxfm2dParam(tx_type, TX_64X16, 20, 2.0));
}
#endif // CONFIG_TX64X64
}
......
......@@ -167,8 +167,8 @@ vector<AV1InvTxfm2dParam> GetInvTxfm2dParamList() {
param_list.push_back(AV1InvTxfm2dParam(tx_type, TX_64X64, 3, 0.2));
param_list.push_back(AV1InvTxfm2dParam(tx_type, TX_32X64, 3, 0.3));
param_list.push_back(AV1InvTxfm2dParam(tx_type, TX_64X32, 3, 0.31));
param_list.push_back(AV1InvTxfm2dParam(tx_type, TX_16X64, 2, 0.11));
param_list.push_back(AV1InvTxfm2dParam(tx_type, TX_64X16, 2, 0.11));
param_list.push_back(AV1InvTxfm2dParam(tx_type, TX_16X64, 2, 0.16));
param_list.push_back(AV1InvTxfm2dParam(tx_type, TX_64X16, 2, 0.16));
}
#endif // CONFIG_TX64X64
}
......
......@@ -124,8 +124,7 @@ double get_amplification_factor(TX_TYPE tx_type, TX_SIZE tx_size) {
amplify_factor *= pow(2, 0.5);
} else if (abs(rect_type) == 2) {
const int tx_max_dim = AOMMAX(tx_width, tx_height);
const int rect_type2_shift =
tx_max_dim == 64 ? 3 : tx_max_dim == 32 ? 2 : 1;
const int rect_type2_shift = (tx_max_dim >= 32) ? 2 : 1;
amplify_factor *= pow(2, rect_type2_shift);
}
return amplify_factor;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment