Make chroma loopfiltering tx_sizes consistent

Removes existing inconsistencies between chroma tx_sizes
used for chroma loopfiltering.
Includes various refactoring to remove the uv_txsize_lookup
array eventually.

BUG=aomedia:1090

Change-Id: Ib74299b41280ca3ebeaf9a9293242d531d68ad28

av1/common/av1_loopfilter.c

@@ -879,12 +879,10 @@ static void build_masks(AV1_COMMON *const cm,
   const TX_SIZE tx_size_y = txsize_sqr_map[mbmi->tx_size];
   const TX_SIZE tx_size_y_left = txsize_horz_map[mbmi->tx_size];
   const TX_SIZE tx_size_y_above = txsize_vert_map[mbmi->tx_size];
-  const TX_SIZE tx_size_uv =
-      txsize_sqr_map[uv_txsize_lookup[block_size][mbmi->tx_size][1][1]];
-  const TX_SIZE tx_size_uv_left =
-      txsize_horz_map[uv_txsize_lookup[block_size][mbmi->tx_size][1][1]];
-  const TX_SIZE tx_size_uv_above =
-      txsize_vert_map[uv_txsize_lookup[block_size][mbmi->tx_size][1][1]];
+  const TX_SIZE tx_size_uv_actual = av1_get_uv_tx_size(mbmi, 1, 1);
+  const TX_SIZE tx_size_uv = txsize_sqr_map[tx_size_uv_actual];
+  const TX_SIZE tx_size_uv_left = txsize_horz_map[tx_size_uv_actual];
+  const TX_SIZE tx_size_uv_above = txsize_vert_map[tx_size_uv_actual];
 #if CONFIG_EXT_DELTA_Q
 #if CONFIG_LOOPFILTER_LEVEL
   const int filter_level = get_filter_level(cm, lfi_n, 0, 0, mbmi);
@@ -1439,7 +1437,8 @@ static void get_filter_level_and_masks_non420(
       const BLOCK_SIZE bsize = get_plane_block_size(mbmi->sb_type, plane);
       const TX_SIZE mb_tx_size = mbmi->inter_tx_size[tx_row_idx][tx_col_idx];
       tx_size = (plane->plane_type == PLANE_TYPE_UV)
-                    ? uv_txsize_lookup[bsize][mb_tx_size][0][0]
+                    ? av1_get_uv_tx_size_vartx(mbmi, plane, bsize, tx_row_idx,
+                                               tx_col_idx)
                     : mb_tx_size;
     }
 
@@ -2040,9 +2039,9 @@ static TX_SIZE av1_get_transform_size(const MODE_INFO *const mi,
 
     assert(mb_tx_size < TX_SIZES_ALL);
 
-    tx_size = (plane == AOM_PLANE_Y)
-                  ? mb_tx_size
-                  : uv_txsize_lookup[bsize][mb_tx_size][0][0];
+    tx_size = (plane == AOM_PLANE_Y) ? mb_tx_size : av1_get_uv_tx_size_vartx(
+                                                        mbmi, plane_ptr, bsize,
+                                                        tx_row_idx, tx_col_idx);
     assert(tx_size < TX_SIZES_ALL);
   }
 

av1/common/blockd.h

@@ -1153,8 +1153,7 @@ static INLINE TX_SIZE av1_get_uv_tx_size(const MB_MODE_INFO *mbmi, int ss_x,
                                          int ss_y) {
   if (is_inter_block(mbmi)) {
 #if DISABLE_VARTX_FOR_CHROMA
-    if (ss_x || ss_y)
-      return av1_get_max_uv_txsize(mbmi->sb_type, 1, ss_x, ss_y);
+    return av1_get_max_uv_txsize(mbmi->sb_type, 1, ss_x, ss_y);
 #endif  // DISABLE_VARTX_FOR_CHROMA
   } else {
 #if DISABLE_SMLTX_FOR_CHROMA_INTRA
@@ -1179,6 +1178,24 @@ static INLINE TX_SIZE av1_get_tx_size(int plane, const MACROBLOCKD *xd) {
   return av1_get_uv_tx_size(mbmi, pd->subsampling_x, pd->subsampling_y);
 }
 
+// Temporary function to facilitate removal of uv_txsize_lookup if we
+// decide to go with DISABLE_SMLTX_FOR_CHROMA_INTRA = 1
+// TODO(debargha): Clean this up
+static INLINE TX_SIZE av1_get_uv_tx_size_vartx(
+    const MB_MODE_INFO *mbmi, const struct macroblockd_plane *pd,
+    BLOCK_SIZE bsize, int tx_row, int tx_col) {
+#if DISABLE_VARTX_FOR_CHROMA
+  (void)bsize;
+  (void)tx_row;
+  (void)tx_col;
+  return av1_get_max_uv_txsize(mbmi->sb_type, 1, pd->subsampling_x,
+                               pd->subsampling_y);
+#else
+  (void)pd;
+  return uv_txsize_lookup[bsize][mbmi->inter_tx_size[tx_row][tx_col]][0][0];
+#endif  // DISABLE_VARTX_FOR_CHROMA
+}
+
 void av1_reset_skip_context(MACROBLOCKD *xd, int mi_row, int mi_col,
                             BLOCK_SIZE bsize);
 

av1/decoder/decodeframe.c

@@ -215,7 +215,7 @@ static void decode_reconstruct_tx(AV1_COMMON *cm, MACROBLOCKD *const xd,
   const int tx_row = blk_row >> (1 - pd->subsampling_y);
   const int tx_col = blk_col >> (1 - pd->subsampling_x);
   const TX_SIZE plane_tx_size =
-      plane ? uv_txsize_lookup[bsize][mbmi->inter_tx_size[tx_row][tx_col]][0][0]
+      plane ? av1_get_uv_tx_size_vartx(mbmi, pd, bsize, tx_row, tx_col)
             : mbmi->inter_tx_size[tx_row][tx_col];
   // Scale to match transform block unit.
   const int max_blocks_high = max_block_high(xd, plane_bsize, plane);
@@ -225,7 +225,7 @@ static void decode_reconstruct_tx(AV1_COMMON *cm, MACROBLOCKD *const xd,
 
   if (tx_size == plane_tx_size
 #if DISABLE_VARTX_FOR_CHROMA
-      || pd->subsampling_x || pd->subsampling_y
+      || plane
 #endif  // DISABLE_VARTX_FOR_CHROMA
       ) {
     PLANE_TYPE plane_type = get_plane_type(plane);

av1/encoder/bitstream.c

@@ -495,19 +495,18 @@ static void pack_txb_tokens(aom_writer *w, AV1_COMMON *cm, MACROBLOCK *const x,
   const BLOCK_SIZE bsize = txsize_to_bsize[tx_size];
   const int tx_row = blk_row >> (1 - pd->subsampling_y);
   const int tx_col = blk_col >> (1 - pd->subsampling_x);
-  TX_SIZE plane_tx_size;
   const int max_blocks_high = max_block_high(xd, plane_bsize, plane);
   const int max_blocks_wide = max_block_wide(xd, plane_bsize, plane);
 
   if (blk_row >= max_blocks_high || blk_col >= max_blocks_wide) return;
 
-  plane_tx_size =
-      plane ? uv_txsize_lookup[bsize][mbmi->inter_tx_size[tx_row][tx_col]][0][0]
+  const TX_SIZE plane_tx_size =
+      plane ? av1_get_uv_tx_size_vartx(mbmi, pd, bsize, tx_row, tx_col)
             : mbmi->inter_tx_size[tx_row][tx_col];
 
   if (tx_size == plane_tx_size
 #if DISABLE_VARTX_FOR_CHROMA
-      || pd->subsampling_x || pd->subsampling_y
+      || plane
 #endif  // DISABLE_VARTX_FOR_CHROMA
       ) {
     TOKEN_STATS tmp_token_stats;
@@ -557,19 +556,18 @@ static void pack_txb_tokens(aom_writer *w, const TOKENEXTRA **tp,
   const BLOCK_SIZE bsize = txsize_to_bsize[tx_size];
   const int tx_row = blk_row >> (1 - pd->subsampling_y);
   const int tx_col = blk_col >> (1 - pd->subsampling_x);
-  TX_SIZE plane_tx_size;
   const int max_blocks_high = max_block_high(xd, plane_bsize, plane);
   const int max_blocks_wide = max_block_wide(xd, plane_bsize, plane);
 
   if (blk_row >= max_blocks_high || blk_col >= max_blocks_wide) return;
 
-  plane_tx_size =
-      plane ? uv_txsize_lookup[bsize][mbmi->inter_tx_size[tx_row][tx_col]][0][0]
+  const TX_SIZE plane_tx_size =
+      plane ? av1_get_uv_tx_size_vartx(mbmi, pd, bsize, tx_row, tx_col)
             : mbmi->inter_tx_size[tx_row][tx_col];
 
   if (tx_size == plane_tx_size
 #if DISABLE_VARTX_FOR_CHROMA
-      || pd->subsampling_x || pd->subsampling_y
+      || plane
 #endif  // DISABLE_VARTX_FOR_CHROMA
       ) {
     TOKEN_STATS tmp_token_stats;

av1/encoder/encodemb.c

@@ -658,19 +658,18 @@ static void encode_block_inter(int plane, int block, int blk_row, int blk_col,
   const struct macroblockd_plane *const pd = &xd->plane[plane];
   const int tx_row = blk_row >> (1 - pd->subsampling_y);
   const int tx_col = blk_col >> (1 - pd->subsampling_x);
-  TX_SIZE plane_tx_size;
   const int max_blocks_high = max_block_high(xd, plane_bsize, plane);
   const int max_blocks_wide = max_block_wide(xd, plane_bsize, plane);
 
   if (blk_row >= max_blocks_high || blk_col >= max_blocks_wide) return;
 
-  plane_tx_size =
-      plane ? uv_txsize_lookup[bsize][mbmi->inter_tx_size[tx_row][tx_col]][0][0]
+  const TX_SIZE plane_tx_size =
+      plane ? av1_get_uv_tx_size_vartx(mbmi, pd, bsize, tx_row, tx_col)
             : mbmi->inter_tx_size[tx_row][tx_col];
 
   if (tx_size == plane_tx_size
 #if DISABLE_VARTX_FOR_CHROMA
-      || pd->subsampling_x || pd->subsampling_y
+      || plane
 #endif  // DISABLE_VARTX_FOR_CHROMA
       ) {
     encode_block(plane, block, blk_row, blk_col, plane_bsize, tx_size, arg,

av1/encoder/rdopt.c

@@ -4314,7 +4314,6 @@ static void tx_block_yrd(const AV1_COMP *cpi, MACROBLOCK *x, int blk_row,
   BLOCK_SIZE bsize = txsize_to_bsize[tx_size];
   const int tx_row = blk_row >> (1 - pd->subsampling_y);
   const int tx_col = blk_col >> (1 - pd->subsampling_x);
-  TX_SIZE plane_tx_size;
   const int max_blocks_high = max_block_high(xd, plane_bsize, plane);
   const int max_blocks_wide = max_block_wide(xd, plane_bsize, plane);
 
@@ -4322,8 +4321,8 @@ static void tx_block_yrd(const AV1_COMP *cpi, MACROBLOCK *x, int blk_row,
 
   if (blk_row >= max_blocks_high || blk_col >= max_blocks_wide) return;
 
-  plane_tx_size =
-      plane ? uv_txsize_lookup[bsize][mbmi->inter_tx_size[tx_row][tx_col]][0][0]
+  const TX_SIZE plane_tx_size =
+      plane ? av1_get_uv_tx_size_vartx(mbmi, pd, bsize, tx_row, tx_col)
             : mbmi->inter_tx_size[tx_row][tx_col];
 
   int ctx = txfm_partition_context(tx_above + blk_col, tx_left + blk_row,
@@ -4332,7 +4331,7 @@ static void tx_block_yrd(const AV1_COMP *cpi, MACROBLOCK *x, int blk_row,
   av1_init_rd_stats(rd_stats);
   if (tx_size == plane_tx_size
 #if DISABLE_VARTX_FOR_CHROMA
-      || pd->subsampling_x || pd->subsampling_y
+      || plane
 #endif  // DISABLE_VARTX_FOR_CHROMA
       ) {
     ENTROPY_CONTEXT *ta = above_ctx + blk_col;
@@ -5020,7 +5019,6 @@ static void tx_block_rd(const AV1_COMP *cpi, MACROBLOCK *x, int blk_row,
   BLOCK_SIZE bsize = txsize_to_bsize[tx_size];
   const int tx_row = blk_row >> (1 - pd->subsampling_y);
   const int tx_col = blk_col >> (1 - pd->subsampling_x);
-  TX_SIZE plane_tx_size;
   const int max_blocks_high = max_block_high(xd, plane_bsize, plane);
   const int max_blocks_wide = max_block_wide(xd, plane_bsize, plane);
 
@@ -5028,13 +5026,13 @@ static void tx_block_rd(const AV1_COMP *cpi, MACROBLOCK *x, int blk_row,
 
   if (blk_row >= max_blocks_high || blk_col >= max_blocks_wide) return;
 
-  plane_tx_size =
-      plane ? uv_txsize_lookup[bsize][mbmi->inter_tx_size[tx_row][tx_col]][0][0]
+  const TX_SIZE plane_tx_size =
+      plane ? av1_get_uv_tx_size_vartx(mbmi, pd, bsize, tx_row, tx_col)
             : mbmi->inter_tx_size[tx_row][tx_col];
 
   if (tx_size == plane_tx_size
 #if DISABLE_VARTX_FOR_CHROMA
-      || pd->subsampling_x || pd->subsampling_y
+      || plane
 #endif  // DISABLE_VARTX_FOR_CHROMA
       ) {
     ENTROPY_CONTEXT *ta = above_ctx + blk_col;

av1/encoder/tokenize.c

@@ -508,17 +508,16 @@ void tokenize_vartx(ThreadData *td, TOKENEXTRA **t, RUN_TYPE dry_run,
   const int tx_col = blk_col >> (1 - pd->subsampling_x);
   const int max_blocks_high = max_block_high(xd, plane_bsize, plane);
   const int max_blocks_wide = max_block_wide(xd, plane_bsize, plane);
-  TX_SIZE plane_tx_size;
 
   if (blk_row >= max_blocks_high || blk_col >= max_blocks_wide) return;
 
-  plane_tx_size =
-      plane ? uv_txsize_lookup[bsize][mbmi->inter_tx_size[tx_row][tx_col]][0][0]
+  const TX_SIZE plane_tx_size =
+      plane ? av1_get_uv_tx_size_vartx(mbmi, pd, bsize, tx_row, tx_col)
             : mbmi->inter_tx_size[tx_row][tx_col];
 
   if (tx_size == plane_tx_size
 #if DISABLE_VARTX_FOR_CHROMA
-      || pd->subsampling_x || pd->subsampling_y
+      || plane
 #endif  // DISABLE_VARTX_FOR_CHROMA
       ) {
     plane_bsize = get_plane_block_size(mbmi->sb_type, pd);