Commit ffbff57d authored by Rupert Swarbrick's avatar Rupert Swarbrick

Fix calculation of tx_type cost for rectangular transforms

This patch fixes up av1_tx_type_cost to match the code in
av1_write_tx_type. Beforehand, we wrongly assumed a 32x16 block needed
to signal its transform size (with rect-tx-ext & rect-tx-ext-intra)
because we were passing 16x16 to get_ext_tx_types.

I've also changed av1_write_tx_type to use get_min_tx_size rather than
inlining its body. No functional change, but it's probably better to
use the same helper function both times.

Change-Id: Iff6ee0bff2d332d5270fe0219db88c95e0b051d0
parent 765e34e3
......@@ -1127,7 +1127,7 @@ void av1_write_tx_type(const AV1_COMMON *const cm, const MACROBLOCKD *xd,
#endif
if (!FIXED_TX_TYPE) {
const TX_SIZE square_tx_size = txsize_sqr_map[tx_size];
const TX_SIZE square_tx_size = get_min_tx_size(tx_size);
const BLOCK_SIZE bsize = mbmi->sb_type;
if (get_ext_tx_types(tx_size, bsize, is_inter, cm->reduced_tx_set_used) >
1 &&
......
......@@ -2338,7 +2338,7 @@ int av1_tx_type_cost(const AV1_COMMON *cm, const MACROBLOCK *x,
TX_SIZE tx_size, TX_TYPE tx_type) {
if (plane > 0) return 0;
tx_size = get_min_tx_size(tx_size);
const TX_SIZE square_tx_size = get_min_tx_size(tx_size);
const MB_MODE_INFO *mbmi = &xd->mi[0]->mbmi;
const int is_inter = is_inter_block(mbmi);
......@@ -2348,8 +2348,7 @@ int av1_tx_type_cost(const AV1_COMMON *cm, const MACROBLOCK *x,
get_ext_tx_set(tx_size, bsize, is_inter, cm->reduced_tx_set_used);
if (is_inter) {
if (ext_tx_set > 0)
return x
->inter_tx_type_costs[ext_tx_set][txsize_sqr_map[tx_size]][tx_type];
return x->inter_tx_type_costs[ext_tx_set][square_tx_size][tx_type];
} else {
if (ext_tx_set > 0 && ALLOW_INTRA_EXT_TX) {
#if CONFIG_FILTER_INTRA
......@@ -2359,11 +2358,11 @@ int av1_tx_type_cost(const AV1_COMMON *cm, const MACROBLOCK *x,
.filter_intra_mode[0]];
else
intra_dir = mbmi->mode;
return x->intra_tx_type_costs[ext_tx_set][txsize_sqr_map[tx_size]]
[intra_dir][tx_type];
return x->intra_tx_type_costs[ext_tx_set][square_tx_size][intra_dir]
[tx_type];
#else
return x->intra_tx_type_costs[ext_tx_set][txsize_sqr_map[tx_size]]
[mbmi->mode][tx_type];
return x->intra_tx_type_costs[ext_tx_set][square_tx_size][mbmi->mode]
[tx_type];
#endif
}
}
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment