var_tx+intrabc: Fix infinite recursion

Change-Id: Ibbf0c14934d0bff316cfdf5c252c64b3e68c73e7

av1/decoder/decodeframe.c

@@ -753,6 +753,7 @@ static void decode_reconstruct_tx(AV1_COMMON *cm, MACROBLOCKD *const xd,
   } else {
     const TX_SIZE sub_txs = sub_tx_size_map[tx_size];
     const int bsl = tx_size_wide_unit[sub_txs];
+    assert(sub_txs < tx_size);
     int i;
 
     assert(bsl > 0);

av1/decoder/decodemv.c

@@ -1121,6 +1121,16 @@ static void read_intra_frame_mode_info(AV1_COMMON *const cm,
 
       xd->corrupted |=
           !assign_dv(cm, xd, &mbmi->mv[0], &dv_ref, mi_row, mi_col, bsize, r);
+#if CONFIG_VAR_TX
+      // TODO(aconverse@google.com): Evaluate allowing VAR TX on intrabc blocks
+      const int width = block_size_wide[bsize] >> tx_size_wide_log2[0];
+      const int height = block_size_high[bsize] >> tx_size_high_log2[0];
+      int idx, idy;
+      for (idy = 0; idy < height; ++idy)
+        for (idx = 0; idx < width; ++idx)
+          mbmi->inter_tx_size[idy >> 1][idx >> 1] = mbmi->tx_size;
+      mbmi->min_tx_size = get_min_tx_size(mbmi->tx_size);
+#endif  // CONFIG_VAR_TX
 #if CONFIG_EXT_TX && !CONFIG_TXK_SEL
       av1_read_tx_type(cm, xd,
 #if CONFIG_SUPERTX

av1/encoder/encodemb.c

@@ -1188,12 +1188,13 @@ static void encode_block_inter(int plane, int block, int blk_row, int blk_col,
   if (tx_size == plane_tx_size) {
     encode_block(plane, block, blk_row, blk_col, plane_bsize, tx_size, arg);
   } else {
+    assert(tx_size < TX_SIZES_ALL);
     const TX_SIZE sub_txs = sub_tx_size_map[tx_size];
+    assert(sub_txs < tx_size);
     // This is the square transform block partition entry point.
     int bsl = tx_size_wide_unit[sub_txs];
     int i;
     assert(bsl > 0);
-    assert(tx_size < TX_SIZES_ALL);
 
     for (i = 0; i < 4; ++i) {
       const int offsetr = blk_row + ((i >> 1) * bsl);

av1/encoder/rdopt.c

@@ -9417,6 +9417,17 @@ static int64_t rd_pick_intrabc_mode_sb(const AV1_COMP *cpi, MACROBLOCK *x,
     mbmi->rd_stats = rd_stats;
 #endif
 
+#if CONFIG_VAR_TX
+    // TODO(aconverse@google.com): Evaluate allowing VAR TX on intrabc blocks
+    const int width = block_size_wide[bsize] >> tx_size_wide_log2[0];
+    const int height = block_size_high[bsize] >> tx_size_high_log2[0];
+    int idx, idy;
+    for (idy = 0; idy < height; ++idy)
+      for (idx = 0; idx < width; ++idx)
+        mbmi->inter_tx_size[idy >> 1][idx >> 1] = mbmi->tx_size;
+    mbmi->min_tx_size = get_min_tx_size(mbmi->tx_size);
+#endif  // CONFIG_VAR_TX
+
     const aom_prob skip_prob = av1_get_skip_prob(cm, xd);
 
     RD_STATS rdc_noskip;