Revert wrong uses of TX_SIZE enum.

Introduced by: https://aomedia-review.googlesource.com/c/aom/+/25181

Change-Id: I1f25178d6b273fbeade4c33f153b5f2bac4a8b99

av1/common/av1_loopfilter.c

@@ -2217,7 +2217,7 @@ static void get_filter_level_and_masks_non420(
     const int skip_border_4x4_r =
         ss_y && mi_row + idx_r >= cm->mi_rows - mi_size_high[BLOCK_8X8];
 
-    TX_SIZE tx_size_mask = 0;
+    int tx_size_mask = 0;
     const int c_step = (c >> ss_x);
     const int r_step = (r >> ss_y);
     const int col_mask = 1 << c_step;

av1/decoder/decodemv.c

@@ -499,7 +499,7 @@ static void read_tx_size_vartx(AV1_COMMON *cm, MACROBLOCKD *xd,
 #endif
 
 static TX_SIZE read_selected_tx_size(AV1_COMMON *cm, MACROBLOCKD *xd,
-                                     TX_SIZE tx_size_cat, aom_reader *r) {
+                                     int32_t tx_size_cat, aom_reader *r) {
   FRAME_COUNTS *counts = xd->counts;
   const int ctx = get_tx_size_context(xd);
   FRAME_CONTEXT *ec_ctx = xd->tile_ctx;

av1/encoder/bitstream.c

@@ -351,8 +351,8 @@ static void write_selected_tx_size(const AV1_COMMON *cm, const MACROBLOCKD *xd,
   if (block_signals_txsize(bsize)) {
     const TX_SIZE tx_size = mbmi->tx_size;
     const int is_inter = is_inter_block(mbmi);
-    const TX_SIZE tx_size_ctx = get_tx_size_context(xd);
-    const TX_SIZE tx_size_cat = is_inter ? inter_tx_size_cat_lookup[bsize]
+    const int tx_size_ctx = get_tx_size_context(xd);
+    const int32_t tx_size_cat = is_inter ? inter_tx_size_cat_lookup[bsize]
                                          : intra_tx_size_cat_lookup[bsize];
     const TX_SIZE coded_tx_size = txsize_sqr_up_map[tx_size];
     const int depth = tx_size_to_depth(coded_tx_size);

av1/encoder/encodeframe.c

@@ -6390,8 +6390,8 @@ static void encode_superblock(const AV1_COMP *const cpi, ThreadData *td,
       if (is_inter) {
         tx_partition_count_update(cm, x, bsize, mi_row, mi_col, td->counts);
       } else {
-        const TX_SIZE tx_size_ctx = get_tx_size_context(xd);
-        const TX_SIZE tx_size_cat = is_inter ? inter_tx_size_cat_lookup[bsize]
+        const int tx_size_ctx = get_tx_size_context(xd);
+        const int32_t tx_size_cat = is_inter ? inter_tx_size_cat_lookup[bsize]
                                              : intra_tx_size_cat_lookup[bsize];
         const TX_SIZE coded_tx_size = txsize_sqr_up_map[tx_size];
         const int depth = tx_size_to_depth(coded_tx_size);
@@ -6399,8 +6399,8 @@ static void encode_superblock(const AV1_COMP *const cpi, ThreadData *td,
         if (tx_size != max_txsize_rect_lookup[bsize]) ++x->txb_split_count;
       }
 #else
-      const TX_SIZE tx_size_ctx = get_tx_size_context(xd);
-      const TX_SIZE tx_size_cat = is_inter ? inter_tx_size_cat_lookup[bsize]
+      const int tx_size_ctx = get_tx_size_context(xd);
+      const int32_t tx_size_cat = is_inter ? inter_tx_size_cat_lookup[bsize]
                                            : intra_tx_size_cat_lookup[bsize];
       const TX_SIZE coded_tx_size = txsize_sqr_up_map[tx_size];
       const int depth = tx_size_to_depth(coded_tx_size);

av1/encoder/rdopt.c

@@ -2212,20 +2212,19 @@ void av1_txfm_rd_in_plane_supertx(MACROBLOCK *x, const AV1_COMP *cpi, int *rate,
 }
 #endif  // CONFIG_SUPERTX
 
-static TX_SIZE tx_size_cost(const AV1_COMP *const cpi,
-                            const MACROBLOCK *const x, BLOCK_SIZE bsize,
-                            TX_SIZE tx_size) {
+static int tx_size_cost(const AV1_COMP *const cpi, const MACROBLOCK *const x,
+                        BLOCK_SIZE bsize, TX_SIZE tx_size) {
   const AV1_COMMON *const cm = &cpi->common;
   const MACROBLOCKD *const xd = &x->e_mbd;
   const MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi;
 
   if (cm->tx_mode == TX_MODE_SELECT && block_signals_txsize(mbmi->sb_type)) {
     const int is_inter = is_inter_block(mbmi);
-    const TX_SIZE tx_size_cat = is_inter ? inter_tx_size_cat_lookup[bsize]
+    const int32_t tx_size_cat = is_inter ? inter_tx_size_cat_lookup[bsize]
                                          : intra_tx_size_cat_lookup[bsize];
     const TX_SIZE coded_tx_size = txsize_sqr_up_map[tx_size];
     const int depth = tx_size_to_depth(coded_tx_size);
-    const TX_SIZE tx_size_ctx = get_tx_size_context(xd);
+    const int tx_size_ctx = get_tx_size_context(xd);
     int r_tx_size = x->tx_size_cost[tx_size_cat][tx_size_ctx][depth];
 #if CONFIG_RECT_TX_EXT && (CONFIG_EXT_TX || CONFIG_VAR_TX)
     if (is_quarter_tx_allowed(xd, mbmi, is_inter) && tx_size != coded_tx_size)

av1/encoder/speed_features.h

@@ -458,7 +458,7 @@ typedef struct SPEED_FEATURES {
 
   // Early termination in transform size search, which only applies while
   // tx_size_search_method is USE_FULL_RD.
-  TX_SIZE tx_size_search_breakout;
+  int tx_size_search_breakout;
 
   // adaptive interp_filter search to allow skip of certain filter types.
   int adaptive_interp_filter_search;