Make RECT_TX(>=8x8) work with VAR_TX

Bitstream syntax:
For a rectangular inter block, 'rect_tx' flag is sent to indicate if
the biggest rect tx is used. If no, continue to decode regular
recursive tx partition.

Change-Id: I127e35cc619b65acb5e9a0717f399cdcdb73fbf0

av1/common/entropymode.c

@@ -857,6 +857,12 @@ static const aom_prob
       },
     };
 
+#if CONFIG_EXT_TX && CONFIG_RECT_TX && CONFIG_VAR_TX
+// the probability of (0) using recursive square tx partition vs.
+// (1) biggest rect tx for 4X8-8X4/8X16-16X8/16X32-32X16 blocks
+static const aom_prob default_rect_tx_prob[TX_SIZES - 1] = { 192, 192, 192 };
+#endif  // CONFIG_EXT_TX && CONFIG_RECT_TX && CONFIG_VAR_TX
+
 int av1_get_palette_color_context(const uint8_t *color_map, int cols, int r,
                                   int c, int n, int *color_order) {
   int i, j, max, max_idx, temp;
@@ -1298,6 +1304,9 @@ static void init_mode_probs(FRAME_CONTEXT *fc) {
   av1_copy(fc->tx_size_probs, default_tx_size_prob);
 #if CONFIG_VAR_TX
   av1_copy(fc->txfm_partition_prob, default_txfm_partition_probs);
+#if CONFIG_EXT_TX && CONFIG_RECT_TX
+  av1_copy(fc->rect_tx_prob, default_rect_tx_prob);
+#endif  // CONFIG_EXT_TX && CONFIG_RECT_TX
 #endif
   av1_copy(fc->skip_probs, default_skip_probs);
 #if CONFIG_REF_MV
@@ -1451,6 +1460,15 @@ void av1_adapt_inter_frame_probs(AV1_COMMON *cm) {
   }
 #endif  // CONFIG_EXT_INTER
 
+#if CONFIG_VAR_TX && CONFIG_EXT_TX && CONFIG_RECT_TX
+  if (cm->tx_mode == TX_MODE_SELECT) {
+    for (i = 0; i < TX_SIZES - 1; ++i) {
+      fc->rect_tx_prob[i] =
+          av1_mode_mv_merge_probs(pre_fc->rect_tx_prob[i], counts->rect_tx[i]);
+    }
+  }
+#endif  // CONFIG_VAR_TX && CONFIG_EXT_TX && CONFIG_RECT_TX
+
   for (i = 0; i < BLOCK_SIZE_GROUPS; i++)
     aom_tree_merge_probs(av1_intra_mode_tree, pre_fc->y_mode_prob[i],
                          counts->y_mode[i], fc->y_mode_prob[i]);
@@ -1478,10 +1496,11 @@ void av1_adapt_intra_frame_probs(AV1_COMMON *cm) {
   }
 
 #if CONFIG_VAR_TX
-  if (cm->tx_mode == TX_MODE_SELECT)
+  if (cm->tx_mode == TX_MODE_SELECT) {
     for (i = 0; i < TXFM_PARTITION_CONTEXTS; ++i)
       fc->txfm_partition_prob[i] = av1_mode_mv_merge_probs(
           pre_fc->txfm_partition_prob[i], counts->txfm_partition[i]);
+  }
 #endif
 
   for (i = 0; i < SKIP_CONTEXTS; ++i)

av1/common/entropymode.h

@@ -96,6 +96,10 @@ typedef struct frame_contexts {
   aom_prob tx_size_probs[TX_SIZES - 1][TX_SIZE_CONTEXTS][TX_SIZES - 1];
 #if CONFIG_VAR_TX
   aom_prob txfm_partition_prob[TXFM_PARTITION_CONTEXTS];
+#if CONFIG_EXT_TX && CONFIG_RECT_TX
+  // TODO(yuec) make this flag harmonize with the original syntax
+  aom_prob rect_tx_prob[TX_SIZES - 1];
+#endif  // CONFIG_EXT_TX && CONFIG_RECT_TX
 #endif
   aom_prob skip_probs[SKIP_CONTEXTS];
 #if CONFIG_REF_MV
@@ -179,6 +183,9 @@ typedef struct FRAME_COUNTS {
   unsigned int tx_size[TX_SIZES - 1][TX_SIZE_CONTEXTS][TX_SIZES];
 #if CONFIG_VAR_TX
   unsigned int txfm_partition[TXFM_PARTITION_CONTEXTS][2];
+#if CONFIG_EXT_TX && CONFIG_RECT_TX
+  unsigned int rect_tx[TX_SIZES - 1][2];
+#endif  // CONFIG_EXT_TX && CONFIG_RECT_TX
 #endif
   unsigned int skip[SKIP_CONTEXTS][2];
 #if CONFIG_REF_MV

av1/common/loopfilter.c

@@ -1246,12 +1246,18 @@ void av1_filter_block_plane_non420(AV1_COMMON *cm,
         tx_size_mask = 0;
 
 #if CONFIG_VAR_TX
-      if (is_inter_block(mbmi) && !mbmi->skip)
-        tx_size =
-            (plane->plane_type == PLANE_TYPE_UV)
-                ? uv_txsize_lookup[sb_type][mbmi->inter_tx_size
-                                                [blk_row][blk_col]][ss_x][ss_y]
-                : mbmi->inter_tx_size[blk_row][blk_col];
+      if (is_inter_block(mbmi) && !mbmi->skip) {
+#if CONFIG_EXT_TX && CONFIG_RECT_TX
+        TX_SIZE mb_tx_size = is_rect_tx(mbmi->tx_size)
+                                 ? mbmi->tx_size
+                                 : mbmi->inter_tx_size[blk_row][blk_col];
+#else
+        TX_SIZE mb_tx_size = mbmi->inter_tx_size[blk_row][blk_col];
+#endif
+        tx_size = (plane->plane_type == PLANE_TYPE_UV)
+                      ? uv_txsize_lookup[sb_type][mb_tx_size][ss_x][ss_y]
+                      : mb_tx_size;
+      }
 
 #if CONFIG_EXT_TX && CONFIG_RECT_TX
       tx_size_r =

av1/decoder/decodeframe.c

@@ -1308,10 +1308,9 @@ static void decode_block(AV1Decoder *const pbi, MACROBLOCKD *const xd,
         const int step = num_4x4_blocks_txsize_lookup[max_tx_size];
         int block = 0;
 #if CONFIG_EXT_TX && CONFIG_RECT_TX
-        const TX_SIZE tx_size =
-            plane ? get_uv_tx_size(mbmi, pd) : mbmi->tx_size;
-
-        if (tx_size >= TX_SIZES) {  // rect txsize is used
+        if (is_rect_tx(mbmi->tx_size)) {
+          const TX_SIZE tx_size =
+              plane ? get_uv_tx_size(mbmi, pd) : mbmi->tx_size;
           const int stepr = num_4x4_blocks_high_txsize_lookup[tx_size];
           const int stepc = num_4x4_blocks_wide_txsize_lookup[tx_size];
           const int max_blocks_wide =
@@ -3491,6 +3490,12 @@ static int read_compressed_header(AV1Decoder *pbi, const uint8_t *data,
 #if CONFIG_VAR_TX
   for (k = 0; k < TXFM_PARTITION_CONTEXTS; ++k)
     av1_diff_update_prob(&r, &fc->txfm_partition_prob[k]);
+#if CONFIG_EXT_TX && CONFIG_RECT_TX
+  if (cm->tx_mode == TX_MODE_SELECT) {
+    for (i = 1; i < TX_SIZES - 1; ++i)
+      av1_diff_update_prob(&r, &fc->rect_tx_prob[i]);
+  }
+#endif  // CONFIG_EXT_TX && CONFIG_RECT_TX
 #endif
 
   for (k = 0; k < SKIP_CONTEXTS; ++k)

av1/decoder/decodemv.c

@@ -1632,13 +1632,34 @@ static void read_inter_frame_mode_info(AV1Decoder *const pbi,
       const int width = num_4x4_blocks_wide_lookup[bsize];
       const int height = num_4x4_blocks_high_lookup[bsize];
       int idx, idy;
-      for (idy = 0; idy < height; idy += bs)
-        for (idx = 0; idx < width; idx += bs)
-          read_tx_size_vartx(cm, xd, mbmi, xd->counts, max_tx_size, idy, idx,
-                             r);
+      int tx_size_cat = inter_tx_size_cat_lookup[bsize];
+#if CONFIG_EXT_TX && CONFIG_RECT_TX
+      int is_rect_tx_allowed = inter_block && is_rect_tx_allowed_bsize(bsize);
+      int use_rect_tx = 0;
+
+      if (is_rect_tx_allowed) {
+        use_rect_tx = aom_read(r, cm->fc->rect_tx_prob[tx_size_cat]);
+        if (xd->counts) {
+          ++xd->counts->rect_tx[tx_size_cat][use_rect_tx];
+        }
+      }
+
+      if (use_rect_tx) {
+        mbmi->tx_size = max_txsize_rect_lookup[bsize];
+        set_txfm_ctxs(mbmi->tx_size, xd->n8_w, xd->n8_h, xd);
+      } else {
+#endif  // CONFIG_EXT_TX && CONFIG_RECT_TX
+        for (idy = 0; idy < height; idy += bs)
+          for (idx = 0; idx < width; idx += bs)
+            read_tx_size_vartx(cm, xd, mbmi, xd->counts, max_tx_size, idy, idx,
+                               r);
+#if CONFIG_EXT_TX && CONFIG_RECT_TX
+      }
+#endif
       if (xd->counts) {
         const int ctx = get_tx_size_context(xd);
-        ++xd->counts->tx_size[max_tx_size - TX_8X8][ctx][mbmi->tx_size];
+        ++xd->counts
+              ->tx_size[tx_size_cat][ctx][txsize_sqr_up_map[mbmi->tx_size]];
       }
     } else {
       if (inter_block)

av1/encoder/bitstream.c

@@ -1121,9 +1121,25 @@ static void pack_inter_mode_mvs(AV1_COMP *cpi, const MODE_INFO *mi,
       const int width = num_4x4_blocks_wide_lookup[bsize];
       const int height = num_4x4_blocks_high_lookup[bsize];
       int idx, idy;
-      for (idy = 0; idy < height; idy += bs)
-        for (idx = 0; idx < width; idx += bs)
-          write_tx_size_vartx(cm, xd, mbmi, max_tx_size, idy, idx, w);
+
+#if CONFIG_EXT_TX && CONFIG_RECT_TX
+      if (is_rect_tx_allowed(mbmi)) {
+        int tx_size_cat = inter_tx_size_cat_lookup[bsize];
+
+        aom_write(w, is_rect_tx(mbmi->tx_size),
+                  cm->fc->rect_tx_prob[tx_size_cat]);
+      }
+
+      if (is_rect_tx(mbmi->tx_size)) {
+        set_txfm_ctxs(mbmi->tx_size, xd->n8_w, xd->n8_h, xd);
+      } else {
+#endif  // CONFIG_EXT_TX && CONFIG_RECT_TX
+        for (idy = 0; idy < height; idy += bs)
+          for (idx = 0; idx < width; idx += bs)
+            write_tx_size_vartx(cm, xd, mbmi, max_tx_size, idy, idx, w);
+#if CONFIG_EXT_TX && CONFIG_RECT_TX
+      }
+#endif  // CONFIG_EXT_TX && CONFIG_RECT_TX
     } else {
       set_txfm_ctxs(mbmi->tx_size, xd->n8_w, xd->n8_h, xd);
       write_selected_tx_size(cm, xd, w);
@@ -1650,7 +1666,7 @@ static void write_modes_b(AV1_COMP *cpi, const TileInfo *const tile,
       TX_SIZE tx_size =
           plane ? get_uv_tx_size(mbmi, &xd->plane[plane]) : mbmi->tx_size;
 
-      if (is_inter_block(mbmi) && tx_size < TX_SIZES) {
+      if (is_inter_block(mbmi) && !is_rect_tx(tx_size)) {
 #else
       if (is_inter_block(mbmi)) {
 #endif
@@ -3271,6 +3287,13 @@ static uint32_t write_compressed_header(AV1_COMP *cpi, uint8_t *data) {
 
 #if CONFIG_VAR_TX
   update_txfm_partition_probs(cm, header_bc, counts);
+#if CONFIG_EXT_TX && CONFIG_RECT_TX
+  if (cm->tx_mode == TX_MODE_SELECT) {
+    for (i = 1; i < TX_SIZES - 1; ++i)
+      av1_cond_prob_diff_update(header_bc, &fc->rect_tx_prob[i],
+                                counts->rect_tx[i]);
+  }
+#endif  // CONFIG_EXT_TX && CONFIG_RECT_TX
 #endif
 
   update_skip_probs(cm, header_bc, counts);

av1/encoder/encodeframe.c

@@ -5031,7 +5031,7 @@ static void encode_superblock(AV1_COMP *cpi, ThreadData *td, TOKENEXTRA **t,
     av1_encode_sb(x, AOMMAX(bsize, BLOCK_8X8));
 #if CONFIG_VAR_TX
 #if CONFIG_EXT_TX && CONFIG_RECT_TX
-    if (mbmi->tx_size >= TX_SIZES)
+    if (is_rect_tx(mbmi->tx_size))
       av1_tokenize_sb(cpi, td, t, !output_enabled, AOMMAX(bsize, BLOCK_8X8));
     else
 #endif
@@ -5054,8 +5054,17 @@ static void encode_superblock(AV1_COMP *cpi, ThreadData *td, TOKENEXTRA **t,
       assert(IMPLIES(is_rect_tx(mbmi->tx_size), is_rect_tx_allowed(mbmi)));
 #endif  // CONFIG_EXT_TX && CONFIG_RECT_TX
 #if CONFIG_VAR_TX
-      if (is_inter)
-        tx_partition_count_update(cm, xd, bsize, mi_row, mi_col, td->counts);
+#if CONFIG_EXT_TX && CONFIG_RECT_TX
+      if (is_rect_tx_allowed(mbmi)) {
+        td->counts->rect_tx[tx_size_cat][is_rect_tx(mbmi->tx_size)]++;
+      }
+      if (!is_rect_tx_allowed(mbmi) || !is_rect_tx(mbmi->tx_size)) {
+#endif  // CONFIG_EXT_TX && CONFIG_RECT_TX
+        if (is_inter)
+          tx_partition_count_update(cm, xd, bsize, mi_row, mi_col, td->counts);
+#if CONFIG_EXT_TX && CONFIG_RECT_TX
+      }
+#endif
 #endif
       ++td->counts->tx_size[tx_size_cat][tx_size_ctx][coded_tx_size];
     } else {
@@ -5118,6 +5127,11 @@ static void encode_superblock(AV1_COMP *cpi, ThreadData *td, TOKENEXTRA **t,
       is_inter_block(mbmi) && !(mbmi->skip || seg_skip)) {
     if (!output_enabled)
       tx_partition_set_contexts(cm, xd, bsize, mi_row, mi_col);
+#if CONFIG_EXT_TX && CONFIG_RECT_TX
+    if (is_rect_tx(mbmi->tx_size)) {
+      set_txfm_ctxs(mbmi->tx_size, xd->n8_w, xd->n8_h, xd);
+    }
+#endif  // CONFIG_EXT_TX && CONFIG_RECT_TX
   } else {
     TX_SIZE tx_size;
     // The new intra coding scheme requires no change of transform size

av1/encoder/encodemb.c

@@ -791,7 +791,7 @@ static void encode_block(int plane, int block, int blk_row, int blk_col,
 #endif
 
 #if CONFIG_VAR_TX
-  // Assert not magic number (uninitialised).
+  // Assert not magic number (uninitialized).
   assert(x->blk_skip[plane][(blk_row << bwl) + blk_col] != 234);
 
   if (x->blk_skip[plane][(blk_row << bwl) + blk_col] == 0) {
@@ -976,9 +976,6 @@ void av1_encode_sb(MACROBLOCK *x, BLOCK_SIZE bsize) {
     int idx, idy;
     int block = 0;
     int step = num_4x4_blocks_txsize_lookup[max_tx_size];
-#if CONFIG_EXT_TX && CONFIG_RECT_TX
-    const TX_SIZE tx_size = plane ? get_uv_tx_size(mbmi, pd) : mbmi->tx_size;
-#endif
     av1_get_entropy_contexts(bsize, TX_4X4, pd, ctx.ta[plane], ctx.tl[plane]);
 #else
     const struct macroblockd_plane *const pd = &xd->plane[plane];
@@ -991,7 +988,7 @@ void av1_encode_sb(MACROBLOCK *x, BLOCK_SIZE bsize) {
 
 #if CONFIG_VAR_TX
 #if CONFIG_EXT_TX && CONFIG_RECT_TX
-    if (tx_size >= TX_SIZES) {
+    if (is_rect_tx(mbmi->tx_size)) {
       av1_foreach_transformed_block_in_plane(xd, bsize, plane, encode_block,
                                              &arg);
     } else {

av1/encoder/rdopt.c

@@ -2814,6 +2814,61 @@ static int64_t rd_pick_intra_sby_mode(AV1_COMP *cpi, MACROBLOCK *x, int *rate,
   return best_rd;
 }
 
+// Return value 0: early termination triggered, no valid rd cost available;
+//              1: rd cost values are valid.
+static int super_block_uvrd(const AV1_COMP *cpi, MACROBLOCK *x, int *rate,
+                            int64_t *distortion, int *skippable, int64_t *sse,
+                            BLOCK_SIZE bsize, int64_t ref_best_rd) {
+  MACROBLOCKD *const xd = &x->e_mbd;
+  MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi;
+  const TX_SIZE uv_tx_size = get_uv_tx_size(mbmi, &xd->plane[1]);
+  int plane;
+  int pnrate = 0, pnskip = 1;
+  int64_t pndist = 0, pnsse = 0;
+  int is_cost_valid = 1;
+
+  if (ref_best_rd < 0) is_cost_valid = 0;
+
+  if (is_inter_block(mbmi) && is_cost_valid) {
+    int plane;
+    for (plane = 1; plane < MAX_MB_PLANE; ++plane)
+      av1_subtract_plane(x, bsize, plane);
+  }
+
+  *rate = 0;
+  *distortion = 0;
+  *sse = 0;
+  *skippable = 1;
+
+  for (plane = 1; plane < MAX_MB_PLANE; ++plane) {
+    txfm_rd_in_plane(x, cpi, &pnrate, &pndist, &pnskip, &pnsse, ref_best_rd,
+                     plane, bsize, uv_tx_size, cpi->sf.use_fast_coef_costing);
+    if (pnrate == INT_MAX) {
+      is_cost_valid = 0;
+      break;
+    }
+    *rate += pnrate;
+    *distortion += pndist;
+    *sse += pnsse;
+    *skippable &= pnskip;
+    if (RDCOST(x->rdmult, x->rddiv, *rate, *distortion) > ref_best_rd &&
+        RDCOST(x->rdmult, x->rddiv, 0, *sse) > ref_best_rd) {
+      is_cost_valid = 0;
+      break;
+    }
+  }
+
+  if (!is_cost_valid) {
+    // reset cost value
+    *rate = INT_MAX;
+    *distortion = INT64_MAX;
+    *sse = INT64_MAX;
+    *skippable = 0;
+  }
+
+  return is_cost_valid;
+}
+
 #if CONFIG_VAR_TX
 void av1_tx_block_rd_b(const AV1_COMP *cpi, MACROBLOCK *x, TX_SIZE tx_size,
                        int blk_row, int blk_col, int plane, int block,
@@ -3196,6 +3251,61 @@ static int64_t select_tx_size_fix_type(const AV1_COMP *cpi, MACROBLOCK *x,
 
   mbmi->tx_type = tx_type;
   inter_block_yrd(cpi, x, rate, dist, skippable, sse, bsize, ref_best_rd);
+#if CONFIG_EXT_TX && CONFIG_RECT_TX
+  if (is_rect_tx_allowed(mbmi)) {
+    int rate_rect_tx, skippable_rect_tx = 0;
+    int64_t dist_rect_tx, sse_rect_tx, rd, rd_rect_tx;
+    int tx_size_cat = inter_tx_size_cat_lookup[bsize];
+    TX_SIZE tx_size = max_txsize_rect_lookup[bsize];
+    TX_SIZE var_tx_size = mbmi->tx_size;
+
+    txfm_rd_in_plane(x, cpi, &rate_rect_tx, &dist_rect_tx, &skippable_rect_tx,
+                     &sse_rect_tx, ref_best_rd, 0, bsize, tx_size,
+                     cpi->sf.use_fast_coef_costing);
+
+    if (*rate != INT_MAX) {
+      *rate += av1_cost_bit(cm->fc->rect_tx_prob[tx_size_cat], 0);
+      if (*skippable) {
+        rd = RDCOST(x->rdmult, x->rddiv, s1, *sse);
+      } else {
+        rd = RDCOST(x->rdmult, x->rddiv, *rate + s0, *dist);
+        if (is_inter && !xd->lossless[xd->mi[0]->mbmi.segment_id] &&
+            !(*skippable))
+          rd = AOMMIN(rd, RDCOST(x->rdmult, x->rddiv, s1, *sse));
+      }
+    } else {
+      rd = INT64_MAX;
+    }
+
+    if (rate_rect_tx != INT_MAX) {
+      rate_rect_tx += av1_cost_bit(cm->fc->rect_tx_prob[tx_size_cat], 1);
+      if (skippable_rect_tx) {
+        rd_rect_tx = RDCOST(x->rdmult, x->rddiv, s1, sse_rect_tx);
+      } else {
+        rd_rect_tx =
+            RDCOST(x->rdmult, x->rddiv, rate_rect_tx + s0, dist_rect_tx);
+        if (is_inter && !xd->lossless[xd->mi[0]->mbmi.segment_id] &&
+            !(skippable_rect_tx))
+          rd_rect_tx =
+              AOMMIN(rd_rect_tx, RDCOST(x->rdmult, x->rddiv, s1, sse_rect_tx));
+      }
+    } else {
+      rd_rect_tx = INT64_MAX;
+    }
+
+    if (rd_rect_tx < rd) {
+      *rate = rate_rect_tx;
+      *dist = dist_rect_tx;
+      *sse = sse_rect_tx;
+      *skippable = skippable_rect_tx;
+      if (!xd->lossless[mbmi->segment_id]) x->blk_skip[0][0] = *skippable;
+      mbmi->tx_size = tx_size;
+      mbmi->inter_tx_size[0][0] = mbmi->tx_size;
+    } else {
+      mbmi->tx_size = var_tx_size;
+    }
+  }
+#endif  // CONFIG_EXT_TX && CONFIG_RECT_TX
 
   if (*rate == INT_MAX) return INT64_MAX;
 
@@ -3409,17 +3519,24 @@ static int inter_block_uvrd(const AV1_COMP *cpi, MACROBLOCK *x, int *rate,
 
   if (ref_best_rd < 0) is_cost_valid = 0;
 
+  *rate = 0;
+  *distortion = 0;
+  *sse = 0;
+  *skippable = 1;
+
+#if CONFIG_EXT_TX && CONFIG_RECT_TX
+  if (is_rect_tx(mbmi->tx_size)) {
+    return super_block_uvrd(cpi, x, rate, distortion, skippable, sse, bsize,
+                            ref_best_rd);
+  }
+#endif  // CONFIG_EXT_TX && CONFIG_RECT_TX
+
   if (is_inter_block(mbmi) && is_cost_valid) {
     int plane;
     for (plane = 1; plane < MAX_MB_PLANE; ++plane)
       av1_subtract_plane(x, bsize, plane);
   }
 
-  *rate = 0;
-  *distortion = 0;
-  *sse = 0;
-  *skippable = 1;
-
   for (plane = 1; plane < MAX_MB_PLANE; ++plane) {
     const struct macroblockd_plane *const pd = &xd->plane[plane];
     const BLOCK_SIZE plane_bsize = get_plane_block_size(bsize, pd);
@@ -3477,61 +3594,6 @@ static int inter_block_uvrd(const AV1_COMP *cpi, MACROBLOCK *x, int *rate,
 }
 #endif  // CONFIG_VAR_TX
 
-// Return value 0: early termination triggered, no valid rd cost available;
-//              1: rd cost values are valid.
-static int super_block_uvrd(const AV1_COMP *cpi, MACROBLOCK *x, int *rate,
-                            int64_t *distortion, int *skippable, int64_t *sse,
-                            BLOCK_SIZE bsize, int64_t ref_best_rd) {
-  MACROBLOCKD *const xd = &x->e_mbd;
-  MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi;
-  const TX_SIZE uv_tx_size = get_uv_tx_size(mbmi, &xd->plane[1]);
-  int plane;
-  int pnrate = 0, pnskip = 1;
-  int64_t pndist = 0, pnsse = 0;
-  int is_cost_valid = 1;
-
-  if (ref_best_rd < 0) is_cost_valid = 0;
-
-  if (is_inter_block(mbmi) && is_cost_valid) {
-    int plane;
-    for (plane = 1; plane < MAX_MB_PLANE; ++plane)
-      av1_subtract_plane(x, bsize, plane);
-  }
-
-  *rate = 0;
-  *distortion = 0;
-  *sse = 0;
-  *skippable = 1;
-
-  for (plane = 1; plane < MAX_MB_PLANE; ++plane) {
-    txfm_rd_in_plane(x, cpi, &pnrate, &pndist, &pnskip, &pnsse, ref_best_rd,
-                     plane, bsize, uv_tx_size, cpi->sf.use_fast_coef_costing);
-    if (pnrate == INT_MAX) {
-      is_cost_valid = 0;
-      break;
-    }
-    *rate += pnrate;
-    *distortion += pndist;
-    *sse += pnsse;
-    *skippable &= pnskip;
-    if (RDCOST(x->rdmult, x->rddiv, *rate, *distortion) > ref_best_rd &&
-        RDCOST(x->rdmult, x->rddiv, 0, *sse) > ref_best_rd) {
-      is_cost_valid = 0;
-      break;
-    }
-  }
-
-  if (!is_cost_valid) {
-    // reset cost value
-    *rate = INT_MAX;
-    *distortion = INT64_MAX;
-    *sse = INT64_MAX;
-    *skippable = 0;
-  }
-
-  return is_cost_valid;
-}
-
 static void rd_pick_palette_intra_sbuv(
     AV1_COMP *cpi, MACROBLOCK *x, int dc_mode_cost,
     PALETTE_MODE_INFO *palette_mode_info, uint8_t *best_palette_color_map,