Move av1_search_txk_type() to rdopt.c

Change-Id: I4f9d014324b35e30f25cae5fa570620249640cf6

av1/encoder/encodetxb.c

@@ -2362,104 +2362,3 @@ void av1_update_txb_context(const AV1_COMP *cpi, ThreadData *td,
     assert(0);
   }
 }
-
-#if CONFIG_TXK_SEL
-int64_t av1_search_txk_type(const AV1_COMP *cpi, MACROBLOCK *x, int plane,
-                            int block, int blk_row, int blk_col,
-                            BLOCK_SIZE plane_bsize, TX_SIZE tx_size,
-                            const ENTROPY_CONTEXT *a, const ENTROPY_CONTEXT *l,
-                            int use_fast_coef_costing, RD_STATS *rd_stats) {
-  const AV1_COMMON *cm = &cpi->common;
-  MACROBLOCKD *xd = &x->e_mbd;
-  MB_MODE_INFO *mbmi = &xd->mi[0]->mbmi;
-  TX_TYPE txk_start = DCT_DCT;
-  TX_TYPE txk_end = x->rd_model ? DCT_DCT : TX_TYPES - 1;
-  TX_TYPE best_tx_type = txk_start;
-  int64_t best_rd = INT64_MAX;
-  uint8_t best_txb_ctx = 0;
-  uint16_t best_eob = 0;
-  RD_STATS best_rd_stats;
-  TX_TYPE tx_type;
-  int rate_cost = 0;
-
-  av1_invalid_rd_stats(&best_rd_stats);
-
-  for (tx_type = txk_start; tx_type <= txk_end; ++tx_type) {
-    if (plane == 0)
-      mbmi->txk_type[(blk_row << MAX_MIB_SIZE_LOG2) + blk_col] = tx_type;
-    TX_TYPE ref_tx_type =
-        av1_get_tx_type(get_plane_type(plane), xd, blk_row, blk_col, tx_size);
-    if (tx_type != ref_tx_type) {
-      // use av1_get_tx_type() to check if the tx_type is valid for the current
-      // mode if it's not, we skip it here.
-      continue;
-    }
-
-    RD_STATS this_rd_stats;
-    av1_invalid_rd_stats(&this_rd_stats);
-    if (cpi->sf.optimize_coefficients != FULL_TRELLIS_OPT) {
-      av1_xform_quant(
-          cm, x, plane, block, blk_row, blk_col, plane_bsize, tx_size,
-          USE_B_QUANT_NO_TRELLIS ? AV1_XFORM_QUANT_B : AV1_XFORM_QUANT_FP);
-    } else {
-      av1_xform_quant(cm, x, plane, block, blk_row, blk_col, plane_bsize,
-                      tx_size, AV1_XFORM_QUANT_FP);
-      av1_optimize_b(cpi, x, plane, blk_row, blk_col, block, plane_bsize,
-                     tx_size, a, l, 1, &rate_cost);
-    }
-    av1_dist_block(cpi, x, plane, plane_bsize, block, blk_row, blk_col, tx_size,
-                   &this_rd_stats.dist, &this_rd_stats.sse,
-                   OUTPUT_HAS_PREDICTED_PIXELS);
-
-    const int eob = x->plane[plane].eobs[block];
-    const SCAN_ORDER *scan_order = get_scan(cm, tx_size, tx_type, mbmi);
-    if (eob)
-      rate_cost +=
-          av1_tx_type_cost(cm, x, xd, mbmi->sb_type, plane, tx_size, tx_type);
-    else
-      rate_cost =
-          av1_cost_coeffs(cpi, x, plane, blk_row, blk_col, block, tx_size,
-                          scan_order, a, l, use_fast_coef_costing);
-    this_rd_stats.rate = rate_cost;
-
-    int64_t rd = RDCOST(x->rdmult, this_rd_stats.rate, this_rd_stats.dist);
-
-    if (rd < best_rd) {
-      best_rd = rd;
-      best_rd_stats = this_rd_stats;
-      best_tx_type = tx_type;
-      best_txb_ctx = x->plane[plane].txb_entropy_ctx[block];
-      best_eob = x->plane[plane].eobs[block];
-    }
-  }
-
-  av1_merge_rd_stats(rd_stats, &best_rd_stats);
-
-  if (best_eob == 0) best_tx_type = DCT_DCT;
-
-  if (plane == 0)
-    mbmi->txk_type[(blk_row << MAX_MIB_SIZE_LOG2) + blk_col] = best_tx_type;
-  x->plane[plane].txb_entropy_ctx[block] = best_txb_ctx;
-  x->plane[plane].eobs[block] = best_eob;
-
-  if (!is_inter_block(mbmi) && best_eob) {
-    // intra mode needs decoded result such that the next transform block
-    // can use it for prediction.
-    if (cpi->sf.optimize_coefficients != FULL_TRELLIS_OPT) {
-      av1_xform_quant(
-          cm, x, plane, block, blk_row, blk_col, plane_bsize, tx_size,
-          USE_B_QUANT_NO_TRELLIS ? AV1_XFORM_QUANT_B : AV1_XFORM_QUANT_FP);
-    } else {
-      av1_xform_quant(cm, x, plane, block, blk_row, blk_col, plane_bsize,
-                      tx_size, AV1_XFORM_QUANT_FP);
-      av1_optimize_b(cpi, x, plane, blk_row, blk_col, block, plane_bsize,
-                     tx_size, a, l, 1, &rate_cost);
-    }
-
-    av1_inverse_transform_block_facade(xd, plane, block, blk_row, blk_col,
-                                       x->plane[plane].eobs[block],
-                                       cm->reduced_tx_set_used);
-  }
-  return best_rd;
-}
-#endif  // CONFIG_TXK_SEL

av1/encoder/encodetxb.h

@@ -94,14 +94,6 @@ void av1_update_and_record_txb_context(int plane, int block, int blk_row,
 void av1_set_coeff_buffer(const AV1_COMP *const cpi, MACROBLOCK *const x,
                           int mi_row, int mi_col);
 
-#if CONFIG_TXK_SEL
-int64_t av1_search_txk_type(const AV1_COMP *cpi, MACROBLOCK *x, int plane,
-                            int block, int blk_row, int blk_col,
-                            BLOCK_SIZE plane_bsize, TX_SIZE tx_size,
-                            const ENTROPY_CONTEXT *a, const ENTROPY_CONTEXT *l,
-                            int use_fast_coef_costing, RD_STATS *rd_stats);
-#endif
-
 int av1_optimize_txb(const AV1_COMP *cpi, MACROBLOCK *x, int plane, int blk_row,
                      int blk_col, int block, TX_SIZE tx_size, TXB_CTX *txb_ctx,
                      int fast_mode, int *rate_cost);

av1/encoder/rdopt.c

@@ -1817,6 +1817,106 @@ void av1_dist_block(const AV1_COMP *cpi, MACROBLOCK *x, int plane,
   }
 }
 
+#if CONFIG_TXK_SEL
+static int64_t search_txk_type(const AV1_COMP *cpi, MACROBLOCK *x, int plane,
+                               int block, int blk_row, int blk_col,
+                               BLOCK_SIZE plane_bsize, TX_SIZE tx_size,
+                               const ENTROPY_CONTEXT *a,
+                               const ENTROPY_CONTEXT *l,
+                               int use_fast_coef_costing, RD_STATS *rd_stats) {
+  const AV1_COMMON *cm = &cpi->common;
+  MACROBLOCKD *xd = &x->e_mbd;
+  MB_MODE_INFO *mbmi = &xd->mi[0]->mbmi;
+  int rate_cost = 0;
+  const int is_inter = is_inter_block(mbmi);
+  TX_TYPE txk_start = DCT_DCT;
+  TX_TYPE txk_end = x->rd_model ? DCT_DCT : TX_TYPES - 1;
+  TX_TYPE best_tx_type = txk_start;
+  int64_t best_rd = INT64_MAX;
+  uint8_t best_txb_ctx = 0;
+  uint16_t best_eob = 0;
+  RD_STATS best_rd_stats;
+  av1_invalid_rd_stats(&best_rd_stats);
+  for (TX_TYPE tx_type = txk_start; tx_type <= txk_end; ++tx_type) {
+    if (plane == 0)
+      mbmi->txk_type[(blk_row << MAX_MIB_SIZE_LOG2) + blk_col] = tx_type;
+    TX_TYPE ref_tx_type =
+        av1_get_tx_type(get_plane_type(plane), xd, blk_row, blk_col, tx_size);
+    if (tx_type != ref_tx_type) {
+      // use av1_get_tx_type() to check if the tx_type is valid for the current
+      // mode if it's not, we skip it here.
+      continue;
+    }
+
+    RD_STATS this_rd_stats;
+    av1_invalid_rd_stats(&this_rd_stats);
+    if (cpi->sf.optimize_coefficients != FULL_TRELLIS_OPT) {
+      av1_xform_quant(
+          cm, x, plane, block, blk_row, blk_col, plane_bsize, tx_size,
+          USE_B_QUANT_NO_TRELLIS ? AV1_XFORM_QUANT_B : AV1_XFORM_QUANT_FP);
+    } else {
+      av1_xform_quant(cm, x, plane, block, blk_row, blk_col, plane_bsize,
+                      tx_size, AV1_XFORM_QUANT_FP);
+      av1_optimize_b(cpi, x, plane, blk_row, blk_col, block, plane_bsize,
+                     tx_size, a, l, 1, &rate_cost);
+    }
+    av1_dist_block(cpi, x, plane, plane_bsize, block, blk_row, blk_col, tx_size,
+                   &this_rd_stats.dist, &this_rd_stats.sse,
+                   OUTPUT_HAS_PREDICTED_PIXELS);
+
+    const int eob = x->plane[plane].eobs[block];
+    const SCAN_ORDER *scan_order = get_scan(cm, tx_size, tx_type, mbmi);
+    if (eob)
+      rate_cost +=
+          av1_tx_type_cost(cm, x, xd, mbmi->sb_type, plane, tx_size, tx_type);
+    else
+      rate_cost =
+          av1_cost_coeffs(cpi, x, plane, blk_row, blk_col, block, tx_size,
+                          scan_order, a, l, use_fast_coef_costing);
+    this_rd_stats.rate = rate_cost;
+
+    int64_t rd = RDCOST(x->rdmult, this_rd_stats.rate, this_rd_stats.dist);
+
+    if (rd < best_rd) {
+      best_rd = rd;
+      best_rd_stats = this_rd_stats;
+      best_tx_type = tx_type;
+      best_txb_ctx = x->plane[plane].txb_entropy_ctx[block];
+      best_eob = x->plane[plane].eobs[block];
+    }
+  }
+
+  av1_merge_rd_stats(rd_stats, &best_rd_stats);
+
+  if (best_eob == 0) best_tx_type = DCT_DCT;
+
+  if (plane == 0)
+    mbmi->txk_type[(blk_row << MAX_MIB_SIZE_LOG2) + blk_col] = best_tx_type;
+  x->plane[plane].txb_entropy_ctx[block] = best_txb_ctx;
+  x->plane[plane].eobs[block] = best_eob;
+
+  if (!is_inter && best_eob) {
+    // intra mode needs decoded result such that the next transform block
+    // can use it for prediction.
+    if (cpi->sf.optimize_coefficients != FULL_TRELLIS_OPT) {
+      av1_xform_quant(
+          cm, x, plane, block, blk_row, blk_col, plane_bsize, tx_size,
+          USE_B_QUANT_NO_TRELLIS ? AV1_XFORM_QUANT_B : AV1_XFORM_QUANT_FP);
+    } else {
+      av1_xform_quant(cm, x, plane, block, blk_row, blk_col, plane_bsize,
+                      tx_size, AV1_XFORM_QUANT_FP);
+      av1_optimize_b(cpi, x, plane, blk_row, blk_col, block, plane_bsize,
+                     tx_size, a, l, 1, &rate_cost);
+    }
+
+    av1_inverse_transform_block_facade(xd, plane, block, blk_row, blk_col,
+                                       x->plane[plane].eobs[block],
+                                       cm->reduced_tx_set_used);
+  }
+  return best_rd;
+}
+#endif  // CONFIG_TXK_SEL
+
 static void block_rd_txfm(int plane, int block, int blk_row, int blk_col,
                           BLOCK_SIZE plane_bsize, TX_SIZE tx_size, void *arg) {
   struct rdcost_block_args *args = arg;
@@ -1935,9 +2035,8 @@ static void block_rd_txfm(int plane, int block, int blk_row, int blk_col,
 
   this_rd_stats.rate = rate_cost;
 #else   // !CONFIG_TXK_SEL
-  av1_search_txk_type(cpi, x, plane, block, blk_row, blk_col, plane_bsize,
-                      tx_size, a, l, args->use_fast_coef_costing,
-                      &this_rd_stats);
+  search_txk_type(cpi, x, plane, block, blk_row, blk_col, plane_bsize, tx_size,
+                  a, l, args->use_fast_coef_costing, &this_rd_stats);
 #endif  // !CONFIG_TXK_SEL
 
 #if CONFIG_CFL
@@ -2438,7 +2537,7 @@ static void choose_tx_size_type_from_rd(const AV1_COMP *const cpi,
     TX_TYPE tx_end = TX_TYPES;
 #if CONFIG_TXK_SEL
     // The tx_type becomes dummy when lv_map is on. The tx_type search will be
-    // performed in av1_search_txk_type()
+    // performed in search_txk_type()
     tx_end = DCT_DCT + 1;
 #endif
     TX_TYPE tx_type;
@@ -3521,8 +3620,8 @@ void av1_tx_block_rd_b(const AV1_COMP *cpi, MACROBLOCK *x, TX_SIZE tx_size,
   const struct macroblock_plane *const p = &x->plane[plane];
   struct macroblockd_plane *const pd = &xd->plane[plane];
 #if CONFIG_TXK_SEL
-  av1_search_txk_type(cpi, x, plane, block, blk_row, blk_col, plane_bsize,
-                      tx_size, a, l, 0, rd_stats);
+  search_txk_type(cpi, x, plane, block, blk_row, blk_col, plane_bsize, tx_size,
+                  a, l, 0, rd_stats);
   return;
 #endif
   // This function is used only for inter