Commit ffbff57d authored by Rupert Swarbrick's avatar Rupert Swarbrick

Fix calculation of tx_type cost for rectangular transforms

This patch fixes up av1_tx_type_cost to match the code in
av1_write_tx_type. Beforehand, we wrongly assumed a 32x16 block needed
to signal its transform size (with rect-tx-ext & rect-tx-ext-intra)
because we were passing 16x16 to get_ext_tx_types.

I've also changed av1_write_tx_type to use get_min_tx_size rather than
inlining its body. No functional change, but it's probably better to
use the same helper function both times.

Change-Id: Iff6ee0bff2d332d5270fe0219db88c95e0b051d0
parent 765e34e3
...@@ -1127,7 +1127,7 @@ void av1_write_tx_type(const AV1_COMMON *const cm, const MACROBLOCKD *xd, ...@@ -1127,7 +1127,7 @@ void av1_write_tx_type(const AV1_COMMON *const cm, const MACROBLOCKD *xd,
#endif #endif
if (!FIXED_TX_TYPE) { if (!FIXED_TX_TYPE) {
const TX_SIZE square_tx_size = txsize_sqr_map[tx_size]; const TX_SIZE square_tx_size = get_min_tx_size(tx_size);
const BLOCK_SIZE bsize = mbmi->sb_type; const BLOCK_SIZE bsize = mbmi->sb_type;
if (get_ext_tx_types(tx_size, bsize, is_inter, cm->reduced_tx_set_used) > if (get_ext_tx_types(tx_size, bsize, is_inter, cm->reduced_tx_set_used) >
1 && 1 &&
......
...@@ -2338,7 +2338,7 @@ int av1_tx_type_cost(const AV1_COMMON *cm, const MACROBLOCK *x, ...@@ -2338,7 +2338,7 @@ int av1_tx_type_cost(const AV1_COMMON *cm, const MACROBLOCK *x,
TX_SIZE tx_size, TX_TYPE tx_type) { TX_SIZE tx_size, TX_TYPE tx_type) {
if (plane > 0) return 0; if (plane > 0) return 0;
tx_size = get_min_tx_size(tx_size); const TX_SIZE square_tx_size = get_min_tx_size(tx_size);
const MB_MODE_INFO *mbmi = &xd->mi[0]->mbmi; const MB_MODE_INFO *mbmi = &xd->mi[0]->mbmi;
const int is_inter = is_inter_block(mbmi); const int is_inter = is_inter_block(mbmi);
...@@ -2348,8 +2348,7 @@ int av1_tx_type_cost(const AV1_COMMON *cm, const MACROBLOCK *x, ...@@ -2348,8 +2348,7 @@ int av1_tx_type_cost(const AV1_COMMON *cm, const MACROBLOCK *x,
get_ext_tx_set(tx_size, bsize, is_inter, cm->reduced_tx_set_used); get_ext_tx_set(tx_size, bsize, is_inter, cm->reduced_tx_set_used);
if (is_inter) { if (is_inter) {
if (ext_tx_set > 0) if (ext_tx_set > 0)
return x return x->inter_tx_type_costs[ext_tx_set][square_tx_size][tx_type];
->inter_tx_type_costs[ext_tx_set][txsize_sqr_map[tx_size]][tx_type];
} else { } else {
if (ext_tx_set > 0 && ALLOW_INTRA_EXT_TX) { if (ext_tx_set > 0 && ALLOW_INTRA_EXT_TX) {
#if CONFIG_FILTER_INTRA #if CONFIG_FILTER_INTRA
...@@ -2359,11 +2358,11 @@ int av1_tx_type_cost(const AV1_COMMON *cm, const MACROBLOCK *x, ...@@ -2359,11 +2358,11 @@ int av1_tx_type_cost(const AV1_COMMON *cm, const MACROBLOCK *x,
.filter_intra_mode[0]]; .filter_intra_mode[0]];
else else
intra_dir = mbmi->mode; intra_dir = mbmi->mode;
return x->intra_tx_type_costs[ext_tx_set][txsize_sqr_map[tx_size]] return x->intra_tx_type_costs[ext_tx_set][square_tx_size][intra_dir]
[intra_dir][tx_type]; [tx_type];
#else #else
return x->intra_tx_type_costs[ext_tx_set][txsize_sqr_map[tx_size]] return x->intra_tx_type_costs[ext_tx_set][square_tx_size][mbmi->mode]
[mbmi->mode][tx_type]; [tx_type];
#endif #endif
} }
} }
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment