Add MAX_VARTX_DEPTH macro

Change-Id: I85532cf88f91f0f0cb4d9cb4b2dbda8a181297bf

av1/common/enums.h

@@ -58,6 +58,10 @@ extern "C" {
 #define MAX_TILE_COLS 64
 #endif  // CONFIG_EXT_TILE
 
+#if CONFIG_VAR_TX
+#define MAX_VARTX_DEPTH 2
+#endif
+
 // Bitstream profiles indicated by 2-3 bits in the uncompressed header.
 // 00: Profile 0.  8-bit 4:2:0 only.
 // 10: Profile 1.  8-bit 4:4:4, 4:2:2, and 4:4:0.

av1/decoder/decodemv.c

@@ -294,7 +294,7 @@ static void read_tx_size_vartx(AV1_COMMON *cm, MACROBLOCKD *xd,
 
   if (blk_row >= max_blocks_high || blk_col >= max_blocks_wide) return;
 
-  if (depth == 2) {
+  if (depth == MAX_VARTX_DEPTH) {
     int idx, idy;
     inter_tx_size[0][0] = tx_size;
     for (idy = 0; idy < num_4x4_blocks_high_txsize_lookup[tx_size] / 2; ++idy)

av1/encoder/bitstream.c

@@ -379,7 +379,7 @@ static void write_tx_size_vartx(const AV1_COMMON *cm, const MACROBLOCKD *xd,
 
   if (blk_row >= max_blocks_high || blk_col >= max_blocks_wide) return;
 
-  if (depth == 2) {
+  if (depth == MAX_VARTX_DEPTH) {
     txfm_partition_update(xd->above_txfm_context + tx_col,
                           xd->left_txfm_context + tx_row, tx_size);
     return;

av1/encoder/rdopt.c

@@ -3116,13 +3116,13 @@ static void select_tx_block(const AV1_COMP *cpi, MACROBLOCK *x, int blk_row,
       *skip = 0;
     }
 
-    if (tx_size > TX_4X4 && depth < 2)
+    if (tx_size > TX_4X4 && depth < MAX_VARTX_DEPTH)
       *rate += av1_cost_bit(cpi->common.fc->txfm_partition_prob[ctx], 0);
     this_rd = RDCOST(x->rdmult, x->rddiv, *rate, *dist);
     tmp_eob = p->eobs[block];
   }
 
-  if (tx_size > TX_4X4 && depth < 2) {
+  if (tx_size > TX_4X4 && depth < MAX_VARTX_DEPTH) {
     BLOCK_SIZE bsize = txsize_to_bsize[tx_size];
     int bsl = b_height_log2_lookup[bsize];
     int sub_step = num_4x4_blocks_txsize_lookup[tx_size - 1];