Add default reduced set for ext-tx

Here we create a default set for the ext-tx experiment which
includes:

Intra: {DCT, ADST} X {DCT, ADST} + IDTX (identity in both directions)
Inter: DCT_DCT + IDTX

This set is used if indicated in the uncompressed header.

Change-Id: Ic907ada12b4e0c98cabb7c9650ab99942d0c0859

av1/common/blockd.h

@@ -599,7 +599,9 @@ static INLINE int supertx_enabled(const MB_MODE_INFO *mbmi) {
 static const int num_ext_tx_set_inter[EXT_TX_SETS_INTER] = { 1, 16, 12, 2 };
 static const int num_ext_tx_set_intra[EXT_TX_SETS_INTRA] = { 1, 7, 5 };
 
-static INLINE int get_ext_tx_set(TX_SIZE tx_size, BLOCK_SIZE bs, int is_inter) {
+static INLINE int get_ext_tx_set(TX_SIZE tx_size, BLOCK_SIZE bs, int is_inter,
+                                 int use_default) {
+  if (use_default) return is_inter ? 3 : 2;
   tx_size = txsize_sqr_map[tx_size];
 #if CONFIG_CB4X4
   (void)bs;
@@ -660,9 +662,9 @@ static const int ext_tx_used_inter_1D[EXT_TX_SETS_INTER][TX_TYPES_1D] = {
   { 1, 0, 0, 0 }, { 1, 1, 1, 1 }, { 1, 1, 1, 1 }, { 1, 0, 0, 1 },
 };
 
-static INLINE int get_ext_tx_types(TX_SIZE tx_size, BLOCK_SIZE bs,
-                                   int is_inter) {
-  const int set = get_ext_tx_set(tx_size, bs, is_inter);
+static INLINE int get_ext_tx_types(TX_SIZE tx_size, BLOCK_SIZE bs, int is_inter,
+                                   int use_default) {
+  const int set = get_ext_tx_set(tx_size, bs, is_inter, use_default);
   return is_inter ? num_ext_tx_set_inter[set] : num_ext_tx_set_intra[set];
 }
 

av1/common/onyxc_int.h

@@ -326,6 +326,10 @@ typedef struct AV1Common {
 
   int frame_parallel_decode;  // frame-based threading.
 
+#if CONFIG_EXT_TX
+  int reduced_tx_set_used;
+#endif  // CONFIG_EXT_TX
+
 // Context probabilities for reference frame prediction
 #if CONFIG_EXT_REFS
   MV_REFERENCE_FRAME comp_fwd_ref[FWD_REFS];

av1/decoder/decodeframe.c

@@ -2226,8 +2226,10 @@ static void decode_partition(AV1Decoder *const pbi, MACROBLOCKD *const xd,
       reset_skip_context(xd, bsize);
     } else {
 #if CONFIG_EXT_TX
-      if (get_ext_tx_types(supertx_size, bsize, 1) > 1) {
-        int eset = get_ext_tx_set(supertx_size, bsize, 1);
+      if (get_ext_tx_types(supertx_size, bsize, 1, cm->reduced_tx_set_used) >
+          1) {
+        const int eset =
+            get_ext_tx_set(supertx_size, bsize, 1, cm->reduced_tx_set_used);
         if (eset > 0) {
           txfm = aom_read_tree(r, av1_ext_tx_inter_tree[eset],
                                cm->fc->inter_ext_tx_prob[eset][supertx_size],
@@ -4380,6 +4382,10 @@ static size_t read_uncompressed_header(AV1Decoder *pbi,
   cm->tx_mode = read_tx_mode(cm, xd, rb);
   cm->reference_mode = read_frame_reference_mode(cm, rb);
 
+#if CONFIG_EXT_TX
+  cm->reduced_tx_set_used = aom_rb_read_bit(rb);
+#endif  // CONFIG_EXT_TX
+
   read_tile_info(pbi, rb);
   sz = aom_rb_read_literal(rb, 16);
 

av1/decoder/decodemv.c

@@ -737,7 +737,8 @@ static void read_tx_type(const AV1_COMMON *const cm, MACROBLOCKD *xd,
   if (!FIXED_TX_TYPE) {
 #if CONFIG_EXT_TX
     const TX_SIZE square_tx_size = txsize_sqr_map[tx_size];
-    if (get_ext_tx_types(tx_size, mbmi->sb_type, inter_block) > 1 &&
+    if (get_ext_tx_types(tx_size, mbmi->sb_type, inter_block,
+                         cm->reduced_tx_set_used) > 1 &&
         ((!cm->seg.enabled && cm->base_qindex > 0) ||
          (cm->seg.enabled && xd->qindex[mbmi->segment_id] > 0)) &&
         !mbmi->skip &&
@@ -745,7 +746,8 @@ static void read_tx_type(const AV1_COMMON *const cm, MACROBLOCKD *xd,
         !supertx_enabled &&
 #endif  // CONFIG_SUPERTX
         !segfeature_active(&cm->seg, mbmi->segment_id, SEG_LVL_SKIP)) {
-      int eset = get_ext_tx_set(tx_size, mbmi->sb_type, inter_block);
+      const int eset = get_ext_tx_set(tx_size, mbmi->sb_type, inter_block,
+                                      cm->reduced_tx_set_used);
       FRAME_COUNTS *counts = xd->counts;
 
       if (inter_block) {

av1/encoder/bitstream.c

@@ -1295,7 +1295,8 @@ static void write_tx_type(const AV1_COMMON *const cm, const MACROBLOCKD *xd,
 #if CONFIG_EXT_TX
     const TX_SIZE square_tx_size = txsize_sqr_map[tx_size];
     const BLOCK_SIZE bsize = mbmi->sb_type;
-    if (get_ext_tx_types(tx_size, bsize, is_inter) > 1 &&
+    if (get_ext_tx_types(tx_size, bsize, is_inter, cm->reduced_tx_set_used) >
+            1 &&
         ((!cm->seg.enabled && cm->base_qindex > 0) ||
          (cm->seg.enabled && xd->qindex[mbmi->segment_id] > 0)) &&
         !mbmi->skip &&
@@ -1303,7 +1304,8 @@ static void write_tx_type(const AV1_COMMON *const cm, const MACROBLOCKD *xd,
         !supertx_enabled &&
 #endif  // CONFIG_SUPERTX
         !segfeature_active(&cm->seg, mbmi->segment_id, SEG_LVL_SKIP)) {
-      int eset = get_ext_tx_set(tx_size, bsize, is_inter);
+      const int eset =
+          get_ext_tx_set(tx_size, bsize, is_inter, cm->reduced_tx_set_used);
       if (is_inter) {
         assert(ext_tx_used_inter[eset][mbmi->tx_type]);
         if (eset > 0) {
@@ -2609,8 +2611,10 @@ static void write_modes_sb(AV1_COMP *const cpi, const TileInfo *const tile,
 
     skip = write_skip(cm, xd, mbmi->segment_id_supertx, xd->mi[0], w);
 #if CONFIG_EXT_TX
-    if (get_ext_tx_types(supertx_size, bsize, 1) > 1 && !skip) {
-      int eset = get_ext_tx_set(supertx_size, bsize, 1);
+    if (get_ext_tx_types(supertx_size, bsize, 1, cm->reduced_tx_set_used) > 1 &&
+        !skip) {
+      const int eset =
+          get_ext_tx_set(supertx_size, bsize, 1, cm->reduced_tx_set_used);
       if (eset > 0) {
         av1_write_token(w, av1_ext_tx_inter_tree[eset],
                         cm->fc->inter_ext_tx_prob[eset][supertx_size],
@@ -4485,6 +4489,10 @@ static void write_uncompressed_header(AV1_COMP *cpi,
 #endif  // !CONFIG_REF_ADAPT
   }
 
+#if CONFIG_EXT_TX
+  aom_wb_write_bit(wb, cm->reduced_tx_set_used);
+#endif  // CONFIG_EXT_TX
+
   write_tile_info(cm, wb);
 }
 

av1/encoder/encodeframe.c

@@ -2487,9 +2487,11 @@ static void encode_sb(const AV1_COMP *const cpi, ThreadData *td,
                            [supertx_size][1]++;
         td->counts->supertx_size[supertx_size]++;
 #if CONFIG_EXT_TX
-        if (get_ext_tx_types(supertx_size, bsize, 1) > 1 &&
+        if (get_ext_tx_types(supertx_size, bsize, 1, cm->reduced_tx_set_used) >
+                1 &&
             !xd->mi[0]->mbmi.skip) {
-          int eset = get_ext_tx_set(supertx_size, bsize, 1);
+          const int eset =
+              get_ext_tx_set(supertx_size, bsize, 1, cm->reduced_tx_set_used);
           if (eset > 0) {
             ++td->counts
                   ->inter_ext_tx[eset][supertx_size][xd->mi[0]->mbmi.tx_type];
@@ -5135,6 +5137,11 @@ static void encode_frame_internal(AV1_COMP *cpi) {
 
 void av1_encode_frame(AV1_COMP *cpi) {
   AV1_COMMON *const cm = &cpi->common;
+#if CONFIG_EXT_TX
+  // Indicates whether or not to use a default reduced set for ext-tx
+  // rather than the potential full set of 16 transforms
+  cm->reduced_tx_set_used = 0;
+#endif  // CONFIG_EXT_TX
 
   // In the longer term the encoder should be generalized to match the
   // decoder such that we allow compound where one of the 3 buffers has a
@@ -5797,10 +5804,12 @@ static void encode_superblock(const AV1_COMP *const cpi, ThreadData *td,
     ++td->counts
           ->tx_size_totals[txsize_sqr_map[get_uv_tx_size(mbmi, &xd->plane[1])]];
 #if CONFIG_EXT_TX
-    if (get_ext_tx_types(tx_size, bsize, is_inter) > 1 && cm->base_qindex > 0 &&
-        !mbmi->skip &&
+    if (get_ext_tx_types(tx_size, bsize, is_inter, cm->reduced_tx_set_used) >
+            1 &&
+        cm->base_qindex > 0 && !mbmi->skip &&
         !segfeature_active(&cm->seg, mbmi->segment_id, SEG_LVL_SKIP)) {
-      int eset = get_ext_tx_set(tx_size, bsize, is_inter);
+      const int eset =
+          get_ext_tx_set(tx_size, bsize, is_inter, cm->reduced_tx_set_used);
       if (eset > 0) {
         if (is_inter) {
           ++td->counts
@@ -6684,7 +6693,7 @@ static void rd_supertx_sb(const AV1_COMP *const cpi, ThreadData *td,
   MB_MODE_INFO *mbmi;
   TX_TYPE tx_type, best_tx_nostx;
 #if CONFIG_EXT_TX
-  int ext_tx_set;
+  const int ext_tx_set;
 #endif  // CONFIG_EXT_TX
   int tmp_rate_tx = 0, skip_tx = 0;
   int64_t tmp_dist_tx = 0, rd_tx, bestrd_tx = INT64_MAX;
@@ -6756,7 +6765,7 @@ static void rd_supertx_sb(const AV1_COMP *const cpi, ThreadData *td,
   tx_size = max_txsize_lookup[bsize];
   av1_subtract_plane(x, bsize, 0);
 #if CONFIG_EXT_TX
-  ext_tx_set = get_ext_tx_set(tx_size, bsize, 1);
+  ext_tx_set = get_ext_tx_set(tx_size, bsize, 1, cm->reduced_tx_set_used);
 #endif  // CONFIG_EXT_TX
   for (tx_type = DCT_DCT; tx_type < TX_TYPES; ++tx_type) {
 #if CONFIG_VAR_TX
@@ -6792,7 +6801,7 @@ static void rd_supertx_sb(const AV1_COMP *const cpi, ThreadData *td,
 #endif  // CONFIG_VAR_TX
 
 #if CONFIG_EXT_TX
-    if (get_ext_tx_types(tx_size, bsize, 1) > 1 &&
+    if (get_ext_tx_types(tx_size, bsize, 1, cm->reduced_tx_set_used) > 1 &&
         !xd->lossless[xd->mi[0]->mbmi.segment_id] && this_rate != INT_MAX) {
       if (ext_tx_set > 0)
         this_rate +=

av1/encoder/rdopt.c

@@ -1785,9 +1785,10 @@ static int64_t txfm_yrd(const AV1_COMP *const cpi, MACROBLOCK *x,
                    cpi->sf.use_fast_coef_costing);
   if (rd_stats->rate == INT_MAX) return INT64_MAX;
 #if CONFIG_EXT_TX
-  if (get_ext_tx_types(tx_size, bs, is_inter) > 1 &&
+  if (get_ext_tx_types(tx_size, bs, is_inter, cm->reduced_tx_set_used) > 1 &&
       !xd->lossless[xd->mi[0]->mbmi.segment_id]) {
-    const int ext_tx_set = get_ext_tx_set(tx_size, bs, is_inter);
+    const int ext_tx_set =
+        get_ext_tx_set(tx_size, bs, is_inter, cm->reduced_tx_set_used);
     if (is_inter) {
       if (ext_tx_set > 0)
         rd_stats->rate +=
@@ -1885,7 +1886,8 @@ static int64_t choose_tx_size_fix_type(const AV1_COMP *const cpi, BLOCK_SIZE bs,
   if (evaluate_rect_tx) {
     const TX_SIZE rect_tx_size = max_txsize_rect_lookup[bs];
     RD_STATS this_rd_stats;
-    ext_tx_set = get_ext_tx_set(rect_tx_size, bs, is_inter);
+    ext_tx_set =
+        get_ext_tx_set(rect_tx_size, bs, is_inter, cm->reduced_tx_set_used);
     if ((is_inter && ext_tx_used_inter[ext_tx_set][tx_type]) ||
         (!is_inter && ext_tx_used_intra[ext_tx_set][tx_type])) {
       rd = txfm_yrd(cpi, x, &this_rd_stats, ref_best_rd, bs, tx_type,
@@ -1912,7 +1914,7 @@ static int64_t choose_tx_size_fix_type(const AV1_COMP *const cpi, BLOCK_SIZE bs,
       continue;
     if (max_tx_size >= TX_32X32 && n == TX_4X4) continue;
 #if CONFIG_EXT_TX
-    ext_tx_set = get_ext_tx_set(n, bs, is_inter);
+    ext_tx_set = get_ext_tx_set(n, bs, is_inter, cm->reduced_tx_set_used);
     if (is_inter) {
       if (!ext_tx_used_inter[ext_tx_set][tx_type]) continue;
       if (cpi->sf.tx_type_search.prune_mode > NO_PRUNE) {
@@ -1992,7 +1994,8 @@ static void choose_largest_tx_size(const AV1_COMP *const cpi, MACROBLOCK *x,
   mbmi->min_tx_size = get_min_tx_size(mbmi->tx_size);
 #endif
 #if CONFIG_EXT_TX
-  ext_tx_set = get_ext_tx_set(mbmi->tx_size, bs, is_inter);
+  ext_tx_set =
+      get_ext_tx_set(mbmi->tx_size, bs, is_inter, cm->reduced_tx_set_used);
 #endif  // CONFIG_EXT_TX
 
   if (is_inter && cpi->sf.tx_type_search.prune_mode > NO_PRUNE)
@@ -2002,7 +2005,8 @@ static void choose_largest_tx_size(const AV1_COMP *const cpi, MACROBLOCK *x,
     prune = prune_tx_types(cpi, bs, x, xd, 0);
 #endif
 #if CONFIG_EXT_TX
-  if (get_ext_tx_types(mbmi->tx_size, bs, is_inter) > 1 &&
+  if (get_ext_tx_types(mbmi->tx_size, bs, is_inter, cm->reduced_tx_set_used) >
+          1 &&
       !xd->lossless[mbmi->segment_id]) {
 #if CONFIG_PVQ
     od_rollback_buffer pre_buf, post_buf;
@@ -2039,7 +2043,8 @@ static void choose_largest_tx_size(const AV1_COMP *const cpi, MACROBLOCK *x,
       od_encode_rollback(&x->daala_enc, &pre_buf);
 #endif
       if (this_rd_stats.rate == INT_MAX) continue;
-      if (get_ext_tx_types(mbmi->tx_size, bs, is_inter) > 1) {
+      if (get_ext_tx_types(mbmi->tx_size, bs, is_inter,
+                           cm->reduced_tx_set_used) > 1) {
         if (is_inter) {
           if (ext_tx_set > 0)
             this_rd_stats.rate +=
@@ -3096,8 +3101,10 @@ static int64_t rd_pick_intra_sub_8x8_y_mode(const AV1_COMP *const cpi,
   if (!is_lossless) {
     int rate_tx_type = 0;
 #if CONFIG_EXT_TX
-    if (get_ext_tx_types(tx_size, bsize, 0) > 1) {
-      const int eset = get_ext_tx_set(tx_size, bsize, 0);
+    if (get_ext_tx_types(tx_size, bsize, 0, cpi->common.reduced_tx_set_used) >
+        1) {
+      const int eset =
+          get_ext_tx_set(tx_size, bsize, 0, cpi->common.reduced_tx_set_used);
       rate_tx_type = cpi->intra_tx_type_costs[eset][txsize_sqr_map[tx_size]]
                                              [mbmi->mode][mbmi->tx_type];
     }
@@ -4126,9 +4133,11 @@ static int64_t select_tx_size_fix_type(const AV1_COMP *cpi, MACROBLOCK *x,
           mbmi->min_tx_size, get_min_tx_size(mbmi->inter_tx_size[row][col]));
 
 #if CONFIG_EXT_TX
-  if (get_ext_tx_types(mbmi->min_tx_size, bsize, is_inter) > 1 &&
+  if (get_ext_tx_types(mbmi->min_tx_size, bsize, is_inter,
+                       cm->reduced_tx_set_used) > 1 &&
       !xd->lossless[xd->mi[0]->mbmi.segment_id]) {
-    int ext_tx_set = get_ext_tx_set(mbmi->min_tx_size, bsize, is_inter);
+    const int ext_tx_set = get_ext_tx_set(mbmi->min_tx_size, bsize, is_inter,
+                                          cm->reduced_tx_set_used);
     if (is_inter) {
       if (ext_tx_set > 0)
         rd_stats->rate +=
@@ -4185,7 +4194,8 @@ static void select_tx_type_yrd(const AV1_COMP *cpi, MACROBLOCK *x,
   RD_STATS rd_stats_stack[4];
 #endif
 #if CONFIG_EXT_TX
-  int ext_tx_set = get_ext_tx_set(max_tx_size, bsize, is_inter);
+  const int ext_tx_set =
+      get_ext_tx_set(max_tx_size, bsize, is_inter, cm->reduced_tx_set_used);
 #endif  // CONFIG_EXT_TX
 
   if (is_inter && cpi->sf.tx_type_search.prune_mode > NO_PRUNE)
@@ -11799,8 +11809,10 @@ void av1_rd_pick_inter_mode_sub8x8(const struct AV1_COMP *cpi,
       if (!xd->lossless[mbmi->segment_id]) {
         int rate_tx_type = 0;
 #if CONFIG_EXT_TX
-        if (get_ext_tx_types(mbmi->tx_size, bsize, 1) > 1) {
-          const int eset = get_ext_tx_set(mbmi->tx_size, bsize, 1);
+        if (get_ext_tx_types(mbmi->tx_size, bsize, 1, cm->reduced_tx_set_used) >
+            1) {
+          const int eset =
+              get_ext_tx_set(mbmi->tx_size, bsize, 1, cm->reduced_tx_set_used);
           rate_tx_type =
               cpi->inter_tx_type_costs[eset][mbmi->tx_size][mbmi->tx_type];
         }