Commit 6001e04c authored by Hui Su's avatar Hui Su

Correctly handle transform type cost

Only consider transform type cost when there are more than just one
candidate.

Change-Id: I3b16bfb7f4d37a1826230396f46b741dbae82ac7
parent 6f1c2f7c
......@@ -4898,15 +4898,13 @@ static void select_tx_type_yrd(const AV1_COMP *cpi, MACROBLOCK *x,
if (this_rd_stats.rate != INT_MAX) {
const int ext_tx_set = get_ext_tx_set(
mbmi->min_tx_size, bsize, is_inter, cm->reduced_tx_set_used);
TX_SIZE square_tx_size = txsize_sqr_map[mbmi->min_tx_size];
#if CONFIG_TX64X64
// For TX_64X64, use TX_32X32 to lookup cost
if (square_tx_size == TX_64X64) square_tx_size = TX_32X32;
#endif // CONFIG_TX64X64
this_rd_stats.rate +=
x->inter_tx_type_costs[ext_tx_set][square_tx_size][mbmi->tx_type];
this_rd_stats.rate -=
x->inter_tx_type_costs[ext_tx_set][square_tx_size][tx_type];
if (ext_tx_set > 0) {
const TX_SIZE square_tx_size = txsize_sqr_map[mbmi->min_tx_size];
this_rd_stats.rate +=
x->inter_tx_type_costs[ext_tx_set][square_tx_size][mbmi->tx_type];
this_rd_stats.rate -=
x->inter_tx_type_costs[ext_tx_set][square_tx_size][tx_type];
}
}
}
#endif // CONFIG_TXK_SEL
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment