new_multisymbol: use cdf-based cost of txfm partition syntax

Change-Id: I79ad4261af5f6843fc0c533327baf15c36cec2ad

av1/encoder/block.h

@@ -296,6 +296,9 @@ struct macroblock {
 #if CONFIG_RECT_TX_EXT && (CONFIG_EXT_TX || CONFIG_VAR_TX)
   int quarter_tx_size_cost[2];
 #endif
+#if CONFIG_VAR_TX
+  int txfm_partition_cost[TXFM_PARTITION_CONTEXTS][2];
+#endif
 #if CONFIG_EXT_TX
 #if CONFIG_LGT_FROM_PRED
   int intra_lgt_cost[LGT_SIZES][INTRA_MODES][2];

av1/encoder/encodeframe.c

@@ -4872,6 +4872,9 @@ static void update_txfm_count(MACROBLOCK *x, MACROBLOCKD *xd,
 #endif
   {
     ++counts->txfm_partition[ctx][0];
+#if CONFIG_NEW_MULTISYMBOL
+    update_cdf(xd->tile_ctx->txfm_partition_cdf[ctx], 0, 2);
+#endif
 #if CONFIG_RECT_TX_EXT
     if (tx_size == plane_tx_size)
 #endif
@@ -4884,6 +4887,9 @@ static void update_txfm_count(MACROBLOCK *x, MACROBLOCKD *xd,
     int i;
 
     ++counts->txfm_partition[ctx][1];
+#if CONFIG_NEW_MULTISYMBOL
+    update_cdf(xd->tile_ctx->txfm_partition_cdf[ctx], 1, 2);
+#endif
     ++x->txb_split_count;
 
     if (sub_txs == TX_4X4) {

av1/encoder/rd.c

@@ -241,6 +241,18 @@ void av1_fill_mode_rates(AV1_COMMON *const cm, MACROBLOCK *x,
 #endif
 #endif
 
+#if CONFIG_VAR_TX
+  for (i = 0; i < TXFM_PARTITION_CONTEXTS; ++i) {
+#if CONFIG_NEW_MULTISYMBOL
+    av1_cost_tokens_from_cdf(x->txfm_partition_cost[i],
+                             fc->txfm_partition_cdf[i], NULL);
+#else
+    x->txfm_partition_cost[i][0] = av1_cost_bit(fc->txfm_partition_prob[i], 0);
+    x->txfm_partition_cost[i][1] = av1_cost_bit(fc->txfm_partition_prob[i], 1);
+#endif
+  }
+#endif
+
 #if CONFIG_EXT_TX
 #if CONFIG_LGT_FROM_PRED
   if (LGT_FROM_PRED_INTRA) {

av1/encoder/rdopt.c

@@ -4924,8 +4924,7 @@ static void select_tx_block(const AV1_COMP *cpi, MACROBLOCK *x, int blk_row,
     }
 
     if (tx_size > TX_4X4 && depth < MAX_VARTX_DEPTH)
-      rd_stats->rate +=
-          av1_cost_bit(cpi->common.fc->txfm_partition_prob[ctx], 0);
+      rd_stats->rate += x->txfm_partition_cost[ctx][0];
 #if CONFIG_RECT_TX_EXT
     if (check_qttx) {
       assert(blk_row == 0 && blk_col == 0);
@@ -5018,8 +5017,7 @@ static void select_tx_block(const AV1_COMP *cpi, MACROBLOCK *x, int blk_row,
       av1_merge_rd_stats(&rd_stats_qttx, &rd_stats_tmp);
 
       if (tx_size > TX_4X4 && depth < MAX_VARTX_DEPTH) {
-        rd_stats_qttx.rate +=
-            av1_cost_bit(cpi->common.fc->txfm_partition_prob[ctx], 0);
+        rd_stats_qttx.rate += x->txfm_partition_cost[ctx][0];
       }
       rd_stats_qttx.rate += x->quarter_tx_size_cost[1];
       rd_qttx = RDCOST(x->rdmult, rd_stats_qttx.rate, rd_stats_qttx.dist);
@@ -5060,8 +5058,7 @@ static void select_tx_block(const AV1_COMP *cpi, MACROBLOCK *x, int blk_row,
 #if CONFIG_DIST_8X8
     int sub8x8_eob[4];
 #endif
-    sum_rd_stats.rate =
-        av1_cost_bit(cpi->common.fc->txfm_partition_prob[ctx], 1);
+    sum_rd_stats.rate = x->txfm_partition_cost[ctx][1];
 
     assert(tx_size < TX_SIZES_ALL);
 
@@ -5516,6 +5513,8 @@ static void set_skip_flag(const AV1_COMP *cpi, MACROBLOCK *x,
   memset(x->blk_skip[0], 1, sizeof(uint8_t) * n4);
   rd_stats->skip = 1;
 
+  (void)cpi;
+
   // Rate.
   const int tx_size_ctx = txsize_sqr_map[tx_size];
   ENTROPY_CONTEXT ctxa[2 * MAX_MIB_SIZE];
@@ -5526,7 +5525,7 @@ static void set_skip_flag(const AV1_COMP *cpi, MACROBLOCK *x,
   if (tx_size > TX_4X4) {
     int ctx = txfm_partition_context(
         xd->above_txfm_context, xd->left_txfm_context, mbmi->sb_type, tx_size);
-    rate += av1_cost_bit(cpi->common.fc->txfm_partition_prob[ctx], 0);
+    rate += x->txfm_partition_cost[ctx][0];
   }
 #if !CONFIG_TXK_SEL
 #if CONFIG_EXT_TX