Add min_tx_size variable to recursive transform block partition system

Replace max_tx_size with min_tx_size for transform type decision.

Change-Id: I64e39923a67903d52b381bd93eaac33b3400a201

av1/common/blockd.h

@@ -189,6 +189,7 @@ typedef struct {
   // TODO(jingning): This effectively assigned a separate entry for each
   // 8x8 block. Apparently it takes much more space than needed.
   TX_SIZE inter_tx_size[MAX_MIB_SIZE][MAX_MIB_SIZE];
+  TX_SIZE min_tx_size;
 #endif
   int8_t skip;
   int8_t segment_id;

av1/common/onyxc_int.h

@@ -731,6 +731,11 @@ static INLINE void av1_zero_left_context(MACROBLOCKD *const xd) {
 }
 
 #if CONFIG_VAR_TX
+static INLINE TX_SIZE get_min_tx_size(const TX_SIZE tx_size) {
+  if (tx_size >= TX_SIZES_ALL) assert(0);
+  return txsize_sqr_map[tx_size];
+}
+
 static INLINE void set_txfm_ctx(TXFM_CONTEXT *txfm_ctx, uint8_t txs, int len) {
   int i;
   for (i = 0; i < len; ++i) txfm_ctx[i] = txs;

av1/decoder/decodemv.c

@@ -315,6 +315,7 @@ static void read_tx_size_vartx(AV1_COMMON *cm, MACROBLOCKD *xd,
       for (idx = 0; idx < tx_size_wide_unit[tx_size] / 2; ++idx)
         inter_tx_size[idy][idx] = tx_size;
     mbmi->tx_size = tx_size;
+    mbmi->min_tx_size = AOMMIN(mbmi->min_tx_size, get_min_tx_size(tx_size));
     if (counts) ++counts->txfm_partition[ctx][0];
     txfm_partition_update(xd->above_txfm_context + tx_col,
                           xd->left_txfm_context + tx_row, tx_size);
@@ -333,6 +334,7 @@ static void read_tx_size_vartx(AV1_COMMON *cm, MACROBLOCKD *xd,
     if (tx_size == TX_8X8) {
       inter_tx_size[0][0] = TX_4X4;
       mbmi->tx_size = TX_4X4;
+      mbmi->min_tx_size = get_min_tx_size(mbmi->tx_size);
       txfm_partition_update(xd->above_txfm_context + tx_col,
                             xd->left_txfm_context + tx_row, TX_4X4);
       return;
@@ -352,6 +354,7 @@ static void read_tx_size_vartx(AV1_COMMON *cm, MACROBLOCKD *xd,
       for (idx = 0; idx < tx_size_wide_unit[tx_size] / 2; ++idx)
         inter_tx_size[idy][idx] = tx_size;
     mbmi->tx_size = tx_size;
+    mbmi->min_tx_size = AOMMIN(mbmi->min_tx_size, get_min_tx_size(tx_size));
     if (counts) ++counts->txfm_partition[ctx][0];
     txfm_partition_update(xd->above_txfm_context + tx_col,
                           xd->left_txfm_context + tx_row, tx_size);
@@ -672,7 +675,11 @@ static void read_tx_type(const AV1_COMMON *const cm, MACROBLOCKD *xd,
 #endif
                          aom_reader *r) {
   const int inter_block = is_inter_block(mbmi);
+#if CONFIG_VAR_TX
+  const TX_SIZE tx_size = inter_block ? mbmi->min_tx_size : mbmi->tx_size;
+#else
   const TX_SIZE tx_size = mbmi->tx_size;
+#endif
   if (!FIXED_TX_TYPE) {
 #if CONFIG_EXT_TX
     if (get_ext_tx_types(tx_size, mbmi->sb_type, inter_block) > 1 &&
@@ -1856,6 +1863,7 @@ static void read_inter_frame_mode_info(AV1Decoder *const pbi,
         set_txfm_ctxs(mbmi->tx_size, xd->n8_w, xd->n8_h, xd);
       } else {
 #endif  // CONFIG_EXT_TX && CONFIG_RECT_TX
+        mbmi->min_tx_size = TX_SIZES_ALL;
         for (idy = 0; idy < height; idy += bh)
           for (idx = 0; idx < width; idx += bw)
             read_tx_size_vartx(cm, xd, mbmi, xd->counts, max_tx_size,
@@ -1877,7 +1885,7 @@ static void read_inter_frame_mode_info(AV1Decoder *const pbi,
           for (idx = 0; idx < width; ++idx)
             mbmi->inter_tx_size[idy >> 1][idx >> 1] = mbmi->tx_size;
       }
-
+      mbmi->min_tx_size = get_min_tx_size(mbmi->tx_size);
       set_txfm_ctxs(mbmi->tx_size, xd->n8_w, xd->n8_h, xd);
     }
 #else

av1/encoder/bitstream.c

@@ -1129,7 +1129,11 @@ static void write_tx_type(const AV1_COMMON *const cm,
 #endif
                           aom_writer *w) {
   const int is_inter = is_inter_block(mbmi);
+#if CONFIG_VAR_TX
+  const TX_SIZE tx_size = is_inter ? mbmi->min_tx_size : mbmi->tx_size;
+#else
   const TX_SIZE tx_size = mbmi->tx_size;
+#endif
   if (!FIXED_TX_TYPE) {
 #if CONFIG_EXT_TX
     const BLOCK_SIZE bsize = mbmi->sb_type;
@@ -1184,13 +1188,6 @@ static void write_tx_type(const AV1_COMMON *const cm,
             &ext_tx_encodings[mbmi->tx_type]);
 #endif
       }
-    } else {
-      if (!mbmi->skip) {
-#if CONFIG_SUPERTX
-        if (!supertx_enabled)
-#endif  // CONFIG_SUPERTX
-          assert(mbmi->tx_type == DCT_DCT);
-      }
     }
 #endif  // CONFIG_EXT_TX
   }
@@ -1898,6 +1895,7 @@ static void write_modes_b(AV1_COMP *cpi, const TileInfo *const tile,
       (*tok)++;
     }
   }
+
 #if CONFIG_RD_DEBUG
   for (plane = 0; plane < MAX_MB_PLANE; ++plane) {
     if (m->mbmi.txb_coeff_cost[plane] != txb_coeff_cost[plane]) {

av1/encoder/encodeframe.c

@@ -1300,6 +1300,9 @@ static void update_state_supertx(const AV1_COMP *const cpi, ThreadData *td,
   for (i = 0; i < 1; ++i)
     memcpy(x->blk_skip[i], ctx->blk_skip[i],
            sizeof(uint8_t) * ctx->num_4x4_blk);
+
+  if (!is_inter_block(mbmi) || mbmi->skip)
+    mbmi->min_tx_size = get_min_tx_size(mbmi->tx_size);
 #endif  // CONFIG_VAR_TX
 
 #if CONFIG_VAR_TX
@@ -1502,6 +1505,7 @@ static void update_supertx_param(ThreadData *td, PICK_MODE_CONTEXT *ctx,
   for (i = 0; i < 1; ++i)
     memcpy(ctx->blk_skip[i], x->blk_skip[i],
            sizeof(uint8_t) * ctx->num_4x4_blk);
+  ctx->mic.mbmi.min_tx_size = get_min_tx_size(supertx_size);
 #endif  // CONFIG_VAR_TX
   ctx->mic.mbmi.tx_size = supertx_size;
   ctx->skip = x->skip;
@@ -5195,6 +5199,9 @@ static void encode_superblock(const AV1_COMP *const cpi, ThreadData *td,
       }
     }
 #endif  // CONFIG_PALETTE
+#if CONFIG_VAR_TX
+    mbmi->min_tx_size = get_min_tx_size(mbmi->tx_size);
+#endif
     av1_tokenize_sb(cpi, td, t, dry_run, AOMMAX(bsize, BLOCK_8X8), rate);
   } else {
     int ref;
@@ -5222,6 +5229,7 @@ static void encode_superblock(const AV1_COMP *const cpi, ThreadData *td,
 
     av1_encode_sb((AV1_COMMON *)cm, x, AOMMAX(bsize, BLOCK_8X8));
 #if CONFIG_VAR_TX
+    if (mbmi->skip) mbmi->min_tx_size = get_min_tx_size(mbmi->tx_size);
 #if CONFIG_EXT_TX && CONFIG_RECT_TX
     if (is_rect_tx(mbmi->tx_size))
       av1_tokenize_sb(cpi, td, t, dry_run, AOMMAX(bsize, BLOCK_8X8), rate);
@@ -5235,29 +5243,35 @@ static void encode_superblock(const AV1_COMP *const cpi, ThreadData *td,
   }
 
   if (!dry_run) {
+    const int is_inter = is_inter_block(mbmi);
+#if CONFIG_VAR_TX
+    TX_SIZE tx_size =
+        is_inter && !mbmi->skip ? mbmi->min_tx_size : mbmi->tx_size;
+#else
+    TX_SIZE tx_size = mbmi->tx_size;
+#endif
     if (cm->tx_mode == TX_MODE_SELECT && mbmi->sb_type >= BLOCK_8X8 &&
         !(is_inter_block(mbmi) && (mbmi->skip || seg_skip))) {
-      const int is_inter = is_inter_block(mbmi);
       const int tx_size_ctx = get_tx_size_context(xd);
       const int tx_size_cat = is_inter ? inter_tx_size_cat_lookup[bsize]
                                        : intra_tx_size_cat_lookup[bsize];
-      const TX_SIZE coded_tx_size = txsize_sqr_up_map[mbmi->tx_size];
+      const TX_SIZE coded_tx_size = txsize_sqr_up_map[tx_size];
       const int depth = tx_size_to_depth(coded_tx_size);
 #if CONFIG_EXT_TX && CONFIG_RECT_TX
-      assert(IMPLIES(is_rect_tx(mbmi->tx_size), is_rect_tx_allowed(xd, mbmi)));
+      assert(IMPLIES(is_rect_tx(tx_size), is_rect_tx_allowed(xd, mbmi)));
 #endif  // CONFIG_EXT_TX && CONFIG_RECT_TX
 #if CONFIG_VAR_TX
 #if CONFIG_EXT_TX && CONFIG_RECT_TX
       if (is_rect_tx_allowed(xd, mbmi)) {
-        td->counts->rect_tx[tx_size_cat][is_rect_tx(mbmi->tx_size)]++;
+        td->counts->rect_tx[tx_size_cat][is_rect_tx(tx_size)]++;
       }
-      if (!is_rect_tx_allowed(xd, mbmi) || !is_rect_tx(mbmi->tx_size)) {
+      if (!is_rect_tx_allowed(xd, mbmi) || !is_rect_tx(tx_size)) {
 #endif  // CONFIG_EXT_TX && CONFIG_RECT_TX
         if (is_inter) {
           tx_partition_count_update(cm, x, bsize, mi_row, mi_col, td->counts);
         } else {
           ++td->counts->tx_size[tx_size_cat][tx_size_ctx][depth];
-          if (mbmi->tx_size != max_txsize_lookup[bsize]) ++x->txb_split_count;
+          if (tx_size != max_txsize_lookup[bsize]) ++x->txb_split_count;
         }
 #if CONFIG_EXT_TX && CONFIG_RECT_TX
       }
@@ -5268,57 +5282,57 @@ static void encode_superblock(const AV1_COMP *const cpi, ThreadData *td,
 #endif
     } else {
       int i, j;
-      TX_SIZE tx_size;
+      TX_SIZE intra_tx_size;
       // The new intra coding scheme requires no change of transform size
       if (is_inter_block(&mi->mbmi)) {
         if (xd->lossless[mbmi->segment_id]) {
-          tx_size = TX_4X4;
+          intra_tx_size = TX_4X4;
         } else {
-          tx_size = tx_size_from_tx_mode(bsize, cm->tx_mode, 1);
+          intra_tx_size = tx_size_from_tx_mode(bsize, cm->tx_mode, 1);
         }
 #if CONFIG_EXT_TX && CONFIG_RECT_TX
         ++td->counts->tx_size_implied[max_txsize_lookup[bsize]]
-                                     [txsize_sqr_up_map[mbmi->tx_size]];
+                                     [txsize_sqr_up_map[tx_size]];
 #endif  // CONFIG_EXT_TX && CONFIG_RECT_TX
       } else {
-        tx_size = (bsize >= BLOCK_8X8) ? mbmi->tx_size : TX_4X4;
+        intra_tx_size = (bsize >= BLOCK_8X8) ? tx_size : TX_4X4;
       }
 
       for (j = 0; j < mi_height; j++)
         for (i = 0; i < mi_width; i++)
           if (mi_col + i < cm->mi_cols && mi_row + j < cm->mi_rows)
-            mi_8x8[mis * j + i]->mbmi.tx_size = tx_size;
+            mi_8x8[mis * j + i]->mbmi.tx_size = intra_tx_size;
 
 #if CONFIG_VAR_TX
-      if (mbmi->tx_size != max_txsize_lookup[bsize]) ++x->txb_split_count;
+      mbmi->min_tx_size = get_min_tx_size(intra_tx_size);
+      if (intra_tx_size != max_txsize_lookup[bsize]) ++x->txb_split_count;
 #endif
     }
 
-    ++td->counts->tx_size_totals[txsize_sqr_map[mbmi->tx_size]];
+    ++td->counts->tx_size_totals[txsize_sqr_map[tx_size]];
     ++td->counts
           ->tx_size_totals[txsize_sqr_map[get_uv_tx_size(mbmi, &xd->plane[1])]];
 #if CONFIG_EXT_TX
-    if (get_ext_tx_types(mbmi->tx_size, bsize, is_inter_block(mbmi)) > 1 &&
+    if (get_ext_tx_types(tx_size, bsize, is_inter_block(mbmi)) > 1 &&
         cm->base_qindex > 0 && !mbmi->skip &&
         !segfeature_active(&cm->seg, mbmi->segment_id, SEG_LVL_SKIP)) {
-      int eset = get_ext_tx_set(mbmi->tx_size, bsize, is_inter_block(mbmi));
+      int eset = get_ext_tx_set(tx_size, bsize, is_inter_block(mbmi));
       if (eset > 0) {
         if (is_inter_block(mbmi)) {
-          ++td->counts->inter_ext_tx[eset][txsize_sqr_map[mbmi->tx_size]]
+          ++td->counts->inter_ext_tx[eset][txsize_sqr_map[tx_size]]
                                     [mbmi->tx_type];
         } else {
-          ++td->counts->intra_ext_tx[eset][mbmi->tx_size][mbmi->mode]
-                                    [mbmi->tx_type];
+          ++td->counts->intra_ext_tx[eset][tx_size][mbmi->mode][mbmi->tx_type];
         }
       }
     }
 #else
-    if (mbmi->tx_size < TX_32X32 && cm->base_qindex > 0 && !mbmi->skip &&
+    if (tx_size < TX_32X32 && cm->base_qindex > 0 && !mbmi->skip &&
         !segfeature_active(&cm->seg, mbmi->segment_id, SEG_LVL_SKIP)) {
       if (is_inter_block(mbmi)) {
-        ++td->counts->inter_ext_tx[mbmi->tx_size][mbmi->tx_type];
+        ++td->counts->inter_ext_tx[tx_size][mbmi->tx_type];
       } else {
-        ++td->counts->intra_ext_tx[mbmi->tx_size]
+        ++td->counts->intra_ext_tx[tx_size]
                                   [intra_mode_to_tx_type_context[mbmi->mode]]
                                   [mbmi->tx_type];
       }
@@ -5339,7 +5353,7 @@ static void encode_superblock(const AV1_COMP *const cpi, ThreadData *td,
     if (dry_run) tx_partition_set_contexts(cm, xd, bsize, mi_row, mi_col);
 #endif  // CONFIG_EXT_TX && CONFIG_RECT_TX
   } else {
-    TX_SIZE tx_size;
+    TX_SIZE tx_size = mbmi->tx_size;
     // The new intra coding scheme requires no change of transform size
     if (is_inter_block(mbmi))
 #if CONFIG_EXT_TX && CONFIG_RECT_TX
@@ -5355,7 +5369,7 @@ static void encode_superblock(const AV1_COMP *const cpi, ThreadData *td,
                        max_txsize_lookup[bsize]);
 #endif
     else
-      tx_size = (bsize >= BLOCK_8X8) ? mbmi->tx_size : TX_4X4;
+      tx_size = (bsize >= BLOCK_8X8) ? tx_size : TX_4X4;
     mbmi->tx_size = tx_size;
     set_txfm_ctxs(tx_size, xd->n8_w, xd->n8_h, xd);
   }

av1/encoder/rdopt.c

@@ -1536,7 +1536,9 @@ static void choose_largest_tx_size(const AV1_COMP *const cpi, MACROBLOCK *x,
   *sse = INT64_MAX;
 
   mbmi->tx_size = tx_size_from_tx_mode(bs, cm->tx_mode, is_inter);
-
+#if CONFIG_VAR_TX
+  mbmi->min_tx_size = get_min_tx_size(mbmi->tx_size);
+#endif
 #if CONFIG_EXT_TX
   ext_tx_set = get_ext_tx_set(mbmi->tx_size, bs, is_inter);
 #endif  // CONFIG_EXT_TX
@@ -1665,6 +1667,9 @@ static void choose_smallest_tx_size(const AV1_COMP *const cpi, MACROBLOCK *x,
 
   mbmi->tx_size = TX_4X4;
   mbmi->tx_type = DCT_DCT;
+#if CONFIG_VAR_TX
+  mbmi->min_tx_size = get_min_tx_size(TX_4X4);
+#endif
 
   txfm_rd_in_plane(x, cpi, rate, distortion, skip, sse, ref_best_rd, 0, bs,
                    mbmi->tx_size, cpi->sf.use_fast_coef_costing);
@@ -1716,6 +1721,10 @@ static void choose_tx_size_type_from_rd(const AV1_COMP *const cpi,
   mbmi->tx_size = best_tx;
   mbmi->tx_type = best_tx_type;
 
+#if CONFIG_VAR_TX
+  mbmi->min_tx_size = get_min_tx_size(mbmi->tx_size);
+#endif
+
 #if !CONFIG_EXT_TX
   if (mbmi->tx_size >= TX_32X32) assert(mbmi->tx_type == DCT_DCT);
 #endif
@@ -3246,8 +3255,12 @@ static int64_t select_tx_size_fix_type(const AV1_COMP *cpi, MACROBLOCK *x,
   int s0 = av1_cost_bit(skip_prob, 0);
   int s1 = av1_cost_bit(skip_prob, 1);
   int64_t rd;
+  int row, col;
+  const int max_blocks_high = max_block_high(xd, bsize, 0);
+  const int max_blocks_wide = max_block_wide(xd, bsize, 0);
 
   mbmi->tx_type = tx_type;
+  mbmi->min_tx_size = TX_SIZES_ALL;
   inter_block_yrd(cpi, x, rd_stats, bsize, ref_best_rd);
 #if CONFIG_EXT_TX && CONFIG_RECT_TX
   if (is_rect_tx_allowed(xd, mbmi)) {
@@ -3304,31 +3317,32 @@ static int64_t select_tx_size_fix_type(const AV1_COMP *cpi, MACROBLOCK *x,
 
   if (rd_stats->rate == INT_MAX) return INT64_MAX;
 
+  for (row = 0; row < max_blocks_high / 2; ++row)
+    for (col = 0; col < max_blocks_wide / 2; ++col)
+      mbmi->min_tx_size = AOMMIN(
+          mbmi->min_tx_size, get_min_tx_size(mbmi->inter_tx_size[row][col]));
+
 #if CONFIG_EXT_TX
-  if (get_ext_tx_types(mbmi->tx_size, bsize, is_inter) > 1 &&
+  if (get_ext_tx_types(mbmi->min_tx_size, bsize, is_inter) > 1 &&
       !xd->lossless[xd->mi[0]->mbmi.segment_id]) {
-    int ext_tx_set = get_ext_tx_set(mbmi->tx_size, bsize, is_inter);
+    int ext_tx_set = get_ext_tx_set(mbmi->min_tx_size, bsize, is_inter);
     if (is_inter) {
       if (ext_tx_set > 0)
         rd_stats->rate +=
-            cpi->inter_tx_type_costs[ext_tx_set][txsize_sqr_map[mbmi->tx_size]]
+            cpi->inter_tx_type_costs[ext_tx_set]
+                                    [txsize_sqr_map[mbmi->min_tx_size]]
                                     [mbmi->tx_type];
     } else {
       if (ext_tx_set > 0 && ALLOW_INTRA_EXT_TX)
-        rd_stats->rate += cpi->intra_tx_type_costs[ext_tx_set][mbmi->tx_size]
-                                                  [mbmi->mode][mbmi->tx_type];
+        rd_stats->rate +=
+            cpi->intra_tx_type_costs[ext_tx_set][mbmi->min_tx_size][mbmi->mode]
+                                    [mbmi->tx_type];
     }
   }
 #else   // CONFIG_EXT_TX
-  if (mbmi->tx_size < TX_32X32 && !xd->lossless[xd->mi[0]->mbmi.segment_id]) {
-    if (is_inter)
-      rd_stats->rate += cpi->inter_tx_type_costs[mbmi->tx_size][mbmi->tx_type];
-    else
-      rd_stats->rate +=
-          cpi->intra_tx_type_costs[mbmi->tx_size]
-                                  [intra_mode_to_tx_type_context[mbmi->mode]]
-                                  [mbmi->tx_type];
-  }
+  if (mbmi->min_tx_size < TX_32X32 && !xd->lossless[xd->mi[0]->mbmi.segment_id])
+    rd_stats->rate +=
+        cpi->inter_tx_type_costs[mbmi->min_tx_size][mbmi->tx_type];
 #endif  // CONFIG_EXT_TX
 
   if (rd_stats->skip)
@@ -3355,6 +3369,7 @@ static void select_tx_type_yrd(const AV1_COMP *cpi, MACROBLOCK *x,
   const int is_inter = is_inter_block(mbmi);
   TX_SIZE best_tx_size[MAX_MIB_SIZE][MAX_MIB_SIZE];
   TX_SIZE best_tx = max_txsize_lookup[bsize];
+  TX_SIZE best_min_tx_size = TX_SIZES_ALL;
   uint8_t best_blk_skip[MAX_MIB_SIZE * MAX_MIB_SIZE * 4];
   const int n4 = 1 << (num_pels_log2_lookup[bsize] - 4);
   int idx, idy;
@@ -3388,7 +3403,6 @@ static void select_tx_type_yrd(const AV1_COMP *cpi, MACROBLOCK *x,
       if (!ext_tx_used_intra[ext_tx_set][tx_type]) continue;
     }
 #else   // CONFIG_EXT_TX
-    if (max_tx_size >= TX_32X32 && tx_type != DCT_DCT) continue;
     if (is_inter && cpi->sf.tx_type_search.prune_mode > NO_PRUNE &&
         !do_tx_type_search(tx_type, prune))
       continue;
@@ -3397,6 +3411,9 @@ static void select_tx_type_yrd(const AV1_COMP *cpi, MACROBLOCK *x,
         tx_type != get_default_tx_type(0, xd, 0, max_tx_size))
       continue;
 
+    if (xd->lossless[mbmi->segment_id])
+      if (tx_type != DCT_DCT) continue;
+
     rd = select_tx_size_fix_type(cpi, x, &this_rd_stats, bsize, ref_best_rd,
                                  tx_type);
 
@@ -3405,6 +3422,7 @@ static void select_tx_type_yrd(const AV1_COMP *cpi, MACROBLOCK *x,
       *rd_stats = this_rd_stats;
       best_tx_type = mbmi->tx_type;
       best_tx = mbmi->tx_size;
+      best_min_tx_size = mbmi->min_tx_size;
       memcpy(best_blk_skip, x->blk_skip[0], sizeof(best_blk_skip[0]) * n4);
       for (idy = 0; idy < xd->n8_h; ++idy)
         for (idx = 0; idx < xd->n8_w; ++idx)
@@ -3417,6 +3435,7 @@ static void select_tx_type_yrd(const AV1_COMP *cpi, MACROBLOCK *x,
     for (idx = 0; idx < xd->n8_w; ++idx)
       mbmi->inter_tx_size[idy][idx] = best_tx_size[idy][idx];
   mbmi->tx_size = best_tx;
+  mbmi->min_tx_size = best_min_tx_size;
 #if CONFIG_RD_DEBUG
   // record plane y's transform block coefficient cost
   mbmi->txb_coeff_cost[0] = rd_stats->txb_coeff_cost[0];
@@ -9271,6 +9290,8 @@ void av1_rd_pick_inter_mode_sb(const AV1_COMP *cpi, TileDataEnc *tile_data,
       for (i = 0; i < MAX_MB_PLANE; ++i)
         memcpy(ctx->blk_skip[i], x->blk_skip[i],
                sizeof(uint8_t) * ctx->num_4x4_blk);
+
+      best_mbmode.min_tx_size = mbmi->min_tx_size;
 #endif
       rd_cost->rate += (rate_y + rate_uv - best_rate_y - best_rate_uv);
       rd_cost->dist = dist_y + dist_uv;
@@ -10078,6 +10099,7 @@ void av1_rd_pick_inter_mode_sub8x8(const struct AV1_COMP *cpi,
 
 #if CONFIG_VAR_TX
     mbmi->inter_tx_size[0][0] = mbmi->tx_size;
+    mbmi->min_tx_size = get_min_tx_size(mbmi->tx_size);
 #endif
 
     if (ref_frame == INTRA_FRAME) {