From ffbff57d9da22524dd0e70bc637ada959ccc7ff1 Mon Sep 17 00:00:00 2001 From: Rupert Swarbrick Date: Tue, 12 Dec 2017 11:27:46 +0000 Subject: [PATCH] Fix calculation of tx_type cost for rectangular transforms This patch fixes up av1_tx_type_cost to match the code in av1_write_tx_type. Beforehand, we wrongly assumed a 32x16 block needed to signal its transform size (with rect-tx-ext & rect-tx-ext-intra) because we were passing 16x16 to get_ext_tx_types. I've also changed av1_write_tx_type to use get_min_tx_size rather than inlining its body. No functional change, but it's probably better to use the same helper function both times. Change-Id: Iff6ee0bff2d332d5270fe0219db88c95e0b051d0 --- av1/encoder/bitstream.c | 2 +- av1/encoder/rdopt.c | 13 ++++++------- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/av1/encoder/bitstream.c b/av1/encoder/bitstream.c index 3c305e2f1..df8ba10c0 100644 --- a/av1/encoder/bitstream.c +++ b/av1/encoder/bitstream.c @@ -1127,7 +1127,7 @@ void av1_write_tx_type(const AV1_COMMON *const cm, const MACROBLOCKD *xd, #endif if (!FIXED_TX_TYPE) { - const TX_SIZE square_tx_size = txsize_sqr_map[tx_size]; + const TX_SIZE square_tx_size = get_min_tx_size(tx_size); const BLOCK_SIZE bsize = mbmi->sb_type; if (get_ext_tx_types(tx_size, bsize, is_inter, cm->reduced_tx_set_used) > 1 && diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c index 20c354717..0ee408a6d 100644 --- a/av1/encoder/rdopt.c +++ b/av1/encoder/rdopt.c @@ -2338,7 +2338,7 @@ int av1_tx_type_cost(const AV1_COMMON *cm, const MACROBLOCK *x, TX_SIZE tx_size, TX_TYPE tx_type) { if (plane > 0) return 0; - tx_size = get_min_tx_size(tx_size); + const TX_SIZE square_tx_size = get_min_tx_size(tx_size); const MB_MODE_INFO *mbmi = &xd->mi[0]->mbmi; const int is_inter = is_inter_block(mbmi); @@ -2348,8 +2348,7 @@ int av1_tx_type_cost(const AV1_COMMON *cm, const MACROBLOCK *x, get_ext_tx_set(tx_size, bsize, is_inter, cm->reduced_tx_set_used); if (is_inter) { if (ext_tx_set > 0) - return x - ->inter_tx_type_costs[ext_tx_set][txsize_sqr_map[tx_size]][tx_type]; + return x->inter_tx_type_costs[ext_tx_set][square_tx_size][tx_type]; } else { if (ext_tx_set > 0 && ALLOW_INTRA_EXT_TX) { #if CONFIG_FILTER_INTRA @@ -2359,11 +2358,11 @@ int av1_tx_type_cost(const AV1_COMMON *cm, const MACROBLOCK *x, .filter_intra_mode[0]]; else intra_dir = mbmi->mode; - return x->intra_tx_type_costs[ext_tx_set][txsize_sqr_map[tx_size]] - [intra_dir][tx_type]; + return x->intra_tx_type_costs[ext_tx_set][square_tx_size][intra_dir] + [tx_type]; #else - return x->intra_tx_type_costs[ext_tx_set][txsize_sqr_map[tx_size]] - [mbmi->mode][tx_type]; + return x->intra_tx_type_costs[ext_tx_set][square_tx_size][mbmi->mode] + [tx_type]; #endif } } -- GitLab