Remove tx_size probability tables.

tx_size is transmitted using CDFs now, so these tables are obsolete.

Also, remove a related tx_size_totals table.

Change-Id: I949b46c1c9499bc556e68c77adbdb5aa3288c9a2

av1/common/entropymode.c

@@ -2527,55 +2527,6 @@ static const int palette_color_index_context_lookup[MAX_COLOR_CONTEXT_HASH +
                                                     1] = { -1, -1, 0, -1, -1,
                                                            4,  3,  2, 1 };
 
-// The transform size is coded as an offset to the smallest transform
-// block size.
-const aom_tree_index av1_tx_size_tree[MAX_TX_DEPTH][TREE_SIZE(TX_SIZES)] = {
-  {
-      // Max tx_size is 8X8
-      -0, -1,
-  },
-  {
-      // Max tx_size is 16X16
-      -0, 2, -1, -2,
-  },
-  {
-      // Max tx_size is 32X32
-      -0, 2, -1, 4, -2, -3,
-  },
-#if CONFIG_TX64X64
-  {
-      // Max tx_size is 64X64
-      -0, 2, -1, 4, -2, 6, -3, -4,
-  },
-#endif  // CONFIG_TX64X64
-};
-
-static const aom_prob default_tx_size_prob[MAX_TX_DEPTH][TX_SIZE_CONTEXTS]
-                                          [MAX_TX_DEPTH] = {
-                                            {
-                                                // Max tx_size is 8X8
-                                                { 100 },
-                                                { 66 },
-                                            },
-                                            {
-                                                // Max tx_size is 16X16
-                                                { 20, 152 },
-                                                { 15, 101 },
-                                            },
-                                            {
-                                                // Max tx_size is 32X32
-                                                { 3, 136, 37 },
-                                                { 5, 52, 13 },
-                                            },
-#if CONFIG_TX64X64
-                                            {
-                                                // Max tx_size is 64X64
-                                                { 1, 64, 136, 127 },
-                                                { 1, 32, 52, 67 },
-                                            },
-#endif  // CONFIG_TX64X64
-                                          };
-
 #if CONFIG_RECT_TX_EXT && (CONFIG_EXT_TX || CONFIG_VAR_TX)
 static const aom_prob default_quarter_tx_size_prob = 192;
 #endif
@@ -5012,7 +4963,6 @@ static void init_mode_probs(FRAME_CONTEXT *fc) {
 #if CONFIG_EXT_INTER && CONFIG_COMPOUND_SINGLEREF
   av1_copy(fc->comp_inter_mode_prob, default_comp_inter_mode_p);
 #endif  // CONFIG_EXT_INTER && CONFIG_COMPOUND_SINGLEREF
-  av1_copy(fc->tx_size_probs, default_tx_size_prob);
 #if CONFIG_RECT_TX_EXT && (CONFIG_EXT_TX || CONFIG_VAR_TX)
   fc->quarter_tx_size_prob = default_quarter_tx_size_prob;
 #endif
@@ -5268,17 +5218,12 @@ void av1_adapt_inter_frame_probs(AV1_COMMON *cm) {
 }
 
 void av1_adapt_intra_frame_probs(AV1_COMMON *cm) {
-  int i, j;
+  int i;
   FRAME_CONTEXT *fc = cm->fc;
   const FRAME_CONTEXT *pre_fc = cm->pre_fc;
   const FRAME_COUNTS *counts = &cm->counts;
 
   if (cm->tx_mode == TX_MODE_SELECT) {
-    for (i = 0; i < MAX_TX_DEPTH; ++i) {
-      for (j = 0; j < TX_SIZE_CONTEXTS; ++j)
-        aom_tree_merge_probs(av1_tx_size_tree[i], pre_fc->tx_size_probs[i][j],
-                             counts->tx_size[i][j], fc->tx_size_probs[i][j]);
-    }
 #if CONFIG_RECT_TX_EXT && (CONFIG_EXT_TX || CONFIG_VAR_TX)
     fc->quarter_tx_size_prob = av1_mode_mv_merge_probs(
         pre_fc->quarter_tx_size_prob, counts->quarter_tx_size);

av1/common/entropymode.h

@@ -311,7 +311,6 @@ typedef struct frame_contexts {
 #if CONFIG_EXT_INTER && CONFIG_COMPOUND_SINGLEREF
   aom_prob comp_inter_mode_prob[COMP_INTER_MODE_CONTEXTS];
 #endif  // CONFIG_EXT_INTER && CONFIG_COMPOUND_SINGLEREF
-  aom_prob tx_size_probs[MAX_TX_DEPTH][TX_SIZE_CONTEXTS][MAX_TX_DEPTH];
 #if CONFIG_RECT_TX_EXT && (CONFIG_EXT_TX || CONFIG_VAR_TX)
   aom_prob quarter_tx_size_prob;
 #endif
@@ -492,10 +491,8 @@ typedef struct FRAME_COUNTS {
 #if CONFIG_EXT_INTER && CONFIG_COMPOUND_SINGLEREF
   unsigned int comp_inter_mode[COMP_INTER_MODE_CONTEXTS][2];
 #endif  // CONFIG_EXT_INTER && CONFIG_COMPOUND_SINGLEREF
-  // TODO(any): tx_size_totals is only used by the encoder to decide whether
-  // to use forward updates for the coeff probs, and as such it does not really
-  // belong into this structure.
-  unsigned int tx_size_totals[TX_SIZES];
+  // TODO(urvang): Only needed for !CONFIG_VAR_TX case. So can be removed when
+  // CONFIG_VAR_TX flag is removed.
   unsigned int tx_size[MAX_TX_DEPTH][TX_SIZE_CONTEXTS][MAX_TX_DEPTH + 1];
 #if CONFIG_RECT_TX_EXT && (CONFIG_EXT_TX || CONFIG_VAR_TX)
   unsigned int quarter_tx_size[2];
@@ -576,7 +573,6 @@ extern const aom_tree_index
 #endif
 extern const aom_tree_index
     av1_palette_color_index_tree[PALETTE_SIZES][TREE_SIZE(PALETTE_COLORS)];
-extern const aom_tree_index av1_tx_size_tree[MAX_TX_DEPTH][TREE_SIZE(TX_SIZES)];
 #if CONFIG_EXT_INTRA && CONFIG_INTRA_INTERP
 extern const aom_tree_index av1_intra_filter_tree[TREE_SIZE(INTRA_FILTERS)];
 #endif  // CONFIG_EXT_INTRA && CONFIG_INTRA_INTERP

av1/encoder/encodeframe.c

@@ -6452,9 +6452,6 @@ static void encode_superblock(const AV1_COMP *const cpi, ThreadData *td,
 #endif
     }
 
-    ++td->counts->tx_size_totals[txsize_sqr_map[tx_size]];
-    ++td->counts->tx_size_totals[txsize_sqr_map[av1_get_uv_tx_size(
-        mbmi, &xd->plane[1])]];
 #if !CONFIG_TXK_SEL
     av1_update_tx_type_count(cm, xd, bsize, tx_size, td->counts);
 #endif

tools/aom_entropy_optimizer.c

@@ -678,7 +678,6 @@ int main(int argc, const char **argv) {
 #endif
 
 /* Transform size */
-// TODO(yuec): av1_tx_size_tree has variable sizes, so needs special handling
 #if CONFIG_RECT_TX_EXT && (CONFIG_EXT_TX || CONFIG_VAR_TX)
   cts_each_dim[0] = 2;
   optimize_entropy_table(&fc.quarter_tx_size[0], probsfile, 1, cts_each_dim,