Remove probablity model for tx type

It's not being used anymore, replaced by the CDF model.
The counters are moved under the entropy-stats experiment, so that they
can be used to tune the default CDF values.

Change-Id: I50cb170c4c8699cf7317fd77853b3fedf2342d3e

av1/common/entropymode.h

@@ -322,14 +322,6 @@ typedef struct frame_contexts {
   aom_cdf_prob intrabc_cdf[CDF_SIZE(2)];
 #endif
   int initialized;
-#if CONFIG_EXT_TX
-  aom_prob inter_ext_tx_prob[EXT_TX_SETS_INTER][EXT_TX_SIZES][TX_TYPES - 1];
-  aom_prob intra_ext_tx_prob[EXT_TX_SETS_INTRA][EXT_TX_SIZES][INTRA_MODES]
-                            [TX_TYPES - 1];
-#else
-  aom_prob intra_ext_tx_prob[EXT_TX_SIZES][TX_TYPES][TX_TYPES - 1];
-  aom_prob inter_ext_tx_prob[EXT_TX_SIZES][TX_TYPES - 1];
-#endif  // CONFIG_EXT_TX
 #if CONFIG_SUPERTX
   aom_prob supertx_prob[PARTITION_SUPERTX_CONTEXTS][TX_SIZES];
 #endif  // CONFIG_SUPERTX
@@ -513,10 +505,11 @@ typedef struct FRAME_COUNTS {
   unsigned int delta_lf[DELTA_LF_PROBS][2];
 #endif
 #endif
-#if CONFIG_EXT_TX
-#if CONFIG_RECT_TX
+#if CONFIG_EXT_TX && CONFIG_RECT_TX
   unsigned int tx_size_implied[TX_SIZES][TX_SIZES];
-#endif  // CONFIG_RECT_TX
+#endif  // CONFIG_EXT_TX && CONFIG_RECT_TX
+#if CONFIG_ENTROPY_STATS
+#if CONFIG_EXT_TX
   unsigned int inter_ext_tx[EXT_TX_SETS_INTER][EXT_TX_SIZES][TX_TYPES];
   unsigned int intra_ext_tx[EXT_TX_SETS_INTRA][EXT_TX_SIZES][INTRA_MODES]
                            [TX_TYPES];
@@ -524,6 +517,7 @@ typedef struct FRAME_COUNTS {
   unsigned int intra_ext_tx[EXT_TX_SIZES][TX_TYPES][TX_TYPES];
   unsigned int inter_ext_tx[EXT_TX_SIZES][TX_TYPES];
 #endif  // CONFIG_EXT_TX
+#endif  // CONFIG_ENTROPY_STATS
 #if CONFIG_SUPERTX
   unsigned int supertx[PARTITION_SUPERTX_CONTEXTS][TX_SIZES][2];
   unsigned int supertx_size[TX_SIZES];

av1/decoder/decodeframe.c

@@ -2474,14 +2474,18 @@ static void decode_partition(AV1Decoder *const pbi, MACROBLOCKD *const xd,
               aom_read_symbol(r, ec_ctx->inter_ext_tx_cdf[eset][supertx_size],
                               av1_num_ext_tx_set[tx_set_type], ACCT_STR);
           txfm = av1_ext_tx_inv[tx_set_type][packed_sym];
+#if CONFIG_ENTROPY_STATS
           if (xd->counts) ++xd->counts->inter_ext_tx[eset][supertx_size][txfm];
+#endif  // CONFIG_ENTROPY_STATS
         }
       }
 #else
       if (supertx_size < TX_32X32) {
         txfm = aom_read_symbol(r, ec_ctx->inter_ext_tx_cdf[supertx_size],
                                TX_TYPES, ACCT_STR);
+#if CONFIG_ENTROPY_STATS
         if (xd->counts) ++xd->counts->inter_ext_tx[supertx_size][txfm];
+#endif  // CONFIG_ENTROPY_STATS
       }
 #endif  // CONFIG_EXT_TX
     }
@@ -5240,10 +5244,12 @@ static void debug_check_frame_counts(const AV1_COMMON *const cm) {
       !memcmp(&cm->counts.mv[0], &zero_counts.mv[0], sizeof(cm->counts.mv[0])));
   assert(
       !memcmp(&cm->counts.mv[1], &zero_counts.mv[1], sizeof(cm->counts.mv[0])));
+#if CONFIG_ENTROPY_STATS
   assert(!memcmp(cm->counts.inter_ext_tx, zero_counts.inter_ext_tx,
                  sizeof(cm->counts.inter_ext_tx)));
   assert(!memcmp(cm->counts.intra_ext_tx, zero_counts.intra_ext_tx,
                  sizeof(cm->counts.intra_ext_tx)));
+#endif  // CONFIG_ENTROPY_STATS
 }
 #endif  // NDEBUG
 

av1/decoder/decodemv.c

@@ -992,19 +992,24 @@ void av1_read_tx_type(const AV1_COMMON *const cm, MACROBLOCKD *xd,
       // eset == 0 should correspond to a set with only DCT_DCT and
       // there is no need to read the tx_type
       assert(eset != 0);
+#if CONFIG_ENTROPY_STATS
       FRAME_COUNTS *counts = xd->counts;
-
+#endif  // CONFIG_ENTROPY_STATS
       if (inter_block) {
         *tx_type = av1_ext_tx_inv[tx_set_type][aom_read_symbol(
             r, ec_ctx->inter_ext_tx_cdf[eset][square_tx_size],
             av1_num_ext_tx_set[tx_set_type], ACCT_STR)];
+#if CONFIG_ENTROPY_STATS
         if (counts) ++counts->inter_ext_tx[eset][square_tx_size][*tx_type];
+#endif  // CONFIG_ENTROPY_STATS
       } else if (ALLOW_INTRA_EXT_TX) {
         *tx_type = av1_ext_tx_inv[tx_set_type][aom_read_symbol(
             r, ec_ctx->intra_ext_tx_cdf[eset][square_tx_size][mbmi->mode],
             av1_num_ext_tx_set[tx_set_type], ACCT_STR)];
+#if CONFIG_ENTROPY_STATS
         if (counts)
           ++counts->intra_ext_tx[eset][square_tx_size][mbmi->mode][*tx_type];
+#endif  // CONFIG_ENTROPY_STATS
       }
     } else {
       *tx_type = DCT_DCT;
@@ -1019,18 +1024,23 @@ void av1_read_tx_type(const AV1_COMMON *const cm, MACROBLOCKD *xd,
         !supertx_enabled &&
 #endif  // CONFIG_SUPERTX
         !segfeature_active(&cm->seg, mbmi->segment_id, SEG_LVL_SKIP)) {
+#if CONFIG_ENTROPY_STATS
       FRAME_COUNTS *counts = xd->counts;
-
+#endif  // CONFIG_ENTROPY_STATS
       if (inter_block) {
         *tx_type = av1_ext_tx_inv[aom_read_symbol(
             r, ec_ctx->inter_ext_tx_cdf[tx_size], TX_TYPES, ACCT_STR)];
+#if CONFIG_ENTROPY_STATS
         if (counts) ++counts->inter_ext_tx[tx_size][*tx_type];
+#endif  // CONFIG_ENTROPY_STATS
       } else {
         const TX_TYPE tx_type_nom = intra_mode_to_tx_type_context[mbmi->mode];
         *tx_type = av1_ext_tx_inv[aom_read_symbol(
             r, ec_ctx->intra_ext_tx_cdf[tx_size][tx_type_nom], TX_TYPES,
             ACCT_STR)];
+#if CONFIG_ENTROPY_STATS
         if (counts) ++counts->intra_ext_tx[tx_size][tx_type_nom][*tx_type];
+#endif  // CONFIG_ENTROPY_STATS
       }
     } else {
       *tx_type = DCT_DCT;

av1/encoder/bitstream.c

@@ -76,12 +76,6 @@ static INLINE void write_uniform(aom_writer *w, int n, int v) {
   }
 }
 
-#if CONFIG_EXT_TX
-static struct av1_token ext_tx_inter_encodings[EXT_TX_SETS_INTER][TX_TYPES];
-static struct av1_token ext_tx_intra_encodings[EXT_TX_SETS_INTRA][TX_TYPES];
-#else
-static struct av1_token ext_tx_encodings[TX_TYPES];
-#endif  // CONFIG_EXT_TX
 #if CONFIG_EXT_INTRA
 #if CONFIG_INTRA_INTERP
 static struct av1_token intra_filter_encodings[INTRA_FILTERS];
@@ -120,21 +114,11 @@ static int remux_tiles(const AV1_COMMON *const cm, uint8_t *dst,
 #endif
 void av1_encode_token_init(void) {
 #if CONFIG_EXT_TX
-  int s;
-  for (s = 1; s < EXT_TX_SETS_INTER; ++s) {
-    av1_tokens_from_tree(ext_tx_inter_encodings[s],
-                         av1_ext_tx_tree[av1_ext_tx_set_type_inter[s]]);
-  }
-  for (s = 1; s < EXT_TX_SETS_INTRA; ++s) {
-    av1_tokens_from_tree(ext_tx_intra_encodings[s],
-                         av1_ext_tx_tree[av1_ext_tx_set_type_intra[s]]);
-  }
-  for (s = 1; s < EXT_TX_SET_TYPES; ++s) {
+  for (int s = 1; s < EXT_TX_SET_TYPES; ++s) {
     av1_indices_from_tree(av1_ext_tx_ind[s], av1_ext_tx_inv[s],
                           av1_ext_tx_tree[s]);
   }
 #else
-  av1_tokens_from_tree(ext_tx_encodings, av1_ext_tx_tree);
   /* This hack is necessary because the four TX_TYPES are not consecutive,
       e.g., 0, 1, 2, 3, when doing an in-order traversal of the av1_ext_tx_tree
       structure. */

av1/encoder/encodeframe.c

@@ -2269,6 +2269,7 @@ static void encode_sb(const AV1_COMP *const cpi, ThreadData *td,
         td->counts->supertx[partition_supertx_context_lookup[partition]]
                            [supertx_size][1]++;
         td->counts->supertx_size[supertx_size]++;
+#if CONFIG_ENTROPY_STATS
 #if CONFIG_EXT_TX
         if (get_ext_tx_types(supertx_size, bsize, 1, cm->reduced_tx_set_used) >
                 1 &&
@@ -2285,6 +2286,7 @@ static void encode_sb(const AV1_COMP *const cpi, ThreadData *td,
           ++td->counts->inter_ext_tx[supertx_size][xd->mi[0]->mbmi.tx_type];
         }
 #endif  // CONFIG_EXT_TX
+#endif  // CONFIG_ENTROPY_STATS
       }
 #if CONFIG_EXT_PARTITION_TYPES
       update_ext_partition_context(xd, mi_row, mi_col, subsize, bsize,
@@ -6055,6 +6057,9 @@ void av1_update_tx_type_count(const AV1_COMMON *cm, MACROBLOCKD *xd,
   MB_MODE_INFO *mbmi = &xd->mi[0]->mbmi;
   int is_inter = is_inter_block(mbmi);
   FRAME_CONTEXT *fc = xd->tile_ctx;
+#if !CONFIG_ENTROPY_STATS
+  (void)counts;
+#endif  // !CONFIG_ENTROPY_STATS
 
 #if !CONFIG_TXK_SEL
   TX_TYPE tx_type = mbmi->tx_type;
@@ -6079,10 +6084,14 @@ void av1_update_tx_type_count(const AV1_COMMON *cm, MACROBLOCKD *xd,
         update_cdf(fc->inter_ext_tx_cdf[eset][txsize_sqr_map[tx_size]],
                    av1_ext_tx_ind[tx_set_type][tx_type],
                    av1_num_ext_tx_set[tx_set_type]);
+#if CONFIG_ENTROPY_STATS
         ++counts->inter_ext_tx[eset][txsize_sqr_map[tx_size]][tx_type];
+#endif  // CONFIG_ENTROPY_STATS
       } else {
+#if CONFIG_ENTROPY_STATS
         ++counts->intra_ext_tx[eset][txsize_sqr_map[tx_size]][mbmi->mode]
                               [tx_type];
+#endif  // CONFIG_ENTROPY_STATS
         update_cdf(
             fc->intra_ext_tx_cdf[eset][txsize_sqr_map[tx_size]][mbmi->mode],
             av1_ext_tx_ind[tx_set_type][tx_type],
@@ -6098,12 +6107,16 @@ void av1_update_tx_type_count(const AV1_COMMON *cm, MACROBLOCKD *xd,
       !mbmi->skip &&
       !segfeature_active(&cm->seg, mbmi->segment_id, SEG_LVL_SKIP)) {
     if (is_inter) {
+#if CONFIG_ENTROPY_STATS
       ++counts->inter_ext_tx[tx_size][tx_type];
+#endif  // CONFIG_ENTROPY_STATS
       update_cdf(fc->inter_ext_tx_cdf[tx_size], av1_ext_tx_ind[tx_type],
                  TX_TYPES);
     } else {
+#if CONFIG_ENTROPY_STATS
       ++counts->intra_ext_tx[tx_size][intra_mode_to_tx_type_context[mbmi->mode]]
                             [tx_type];
+#endif  // CONFIG_ENTROPY_STATS
       update_cdf(
           fc->intra_ext_tx_cdf[tx_size]
                               [intra_mode_to_tx_type_context[mbmi->mode]],