Merge "Correction to costing rect_tx" into nextgenv2

3f8b5b90 · Debargha Mukherjee · Gerrit Code Review · 9ac1f7d7 · b85a5a7e · 3f8b5b90
Commit 3f8b5b90 authored Oct 18, 2016 by Debargha Mukherjee Committed by Gerrit Code Review Oct 18, 2016
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -3309,9 +3309,9 @@ static int64_t select_tx_size_fix_type(const AV1_COMP *cpi, MACROBLOCK *x,
    int ext_tx_set = get_ext_tx_set(mbmi->tx_size, bsize, is_inter);
    if (is_inter) {
      if (ext_tx_set > 0)
-        *rate += cpi->inter_tx_type_costs[ext_tx_set]
-                                         [txsize_sqr_up_map[mbmi->tx_size]]
-                                         [mbmi->tx_type];
+        *rate +=
+            cpi->inter_tx_type_costs[ext_tx_set][txsize_sqr_map[mbmi->tx_size]]
+                                    [mbmi->tx_type];
    } else {
      if (ext_tx_set > 0 && ALLOW_INTRA_EXT_TX)
        *rate += cpi->intra_tx_type_costs[ext_tx_set][mbmi->tx_size][mbmi->mode]