Commit 4e7b7d6f authored by Rupert Swarbrick's avatar Rupert Swarbrick Committed by Debargha Mukherjee

Don't work in TX64X64 blocks on a subsampled chroma plane

With ext-partition and tx64x64 enabled, the encoder could choose
TX_64X64 to encode the transform for a subsampled plane of a
BLOCK_128X128 block. This broke an assumption in the nested loop in
write_tokens_b and also caused bug 827 (with a rather cryptic
failure).

This patch changes get_vartx_max_txsize to ensure that the encoder and
decoder don't think they can use TX_64X64 in this situation. It also
adds a couple of assertions to the loop mentioned above so that if
something comes unstuck it'll be much more obvious what went wrong.

BUG=aomedia:827

Change-Id: Ie093f2f20f6242949d68e950c8f95b100867ee17
parent dd8df163
......@@ -1473,13 +1473,28 @@ static INLINE int is_interintra_pred(const MB_MODE_INFO *mbmi) {
#if CONFIG_VAR_TX
static INLINE int get_vartx_max_txsize(const MB_MODE_INFO *const mbmi,
BLOCK_SIZE bsize) {
BLOCK_SIZE bsize, int subsampled) {
#if CONFIG_CB4X4
(void)mbmi;
return max_txsize_rect_lookup[bsize];
TX_SIZE max_txsize = max_txsize_rect_lookup[bsize];
#else
TX_SIZE max_txsize = mbmi->sb_type < BLOCK_8X8
? max_txsize_rect_lookup[mbmi->sb_type]
: max_txsize_rect_lookup[bsize];
#endif // CONFIG_C4X4
return mbmi->sb_type < BLOCK_8X8 ? max_txsize_rect_lookup[mbmi->sb_type]
: max_txsize_rect_lookup[bsize];
#if CONFIG_EXT_PARTITION && CONFIG_TX64X64
// The decoder is designed so that it can process 64x64 luma pixels at a
// time. If this is a chroma plane with subsampling and bsize corresponds to
// a subsampled BLOCK_128X128 then the lookup above will give TX_64X64. That
// mustn't be used for the subsampled plane (because it would be bigger than
// a 64x64 luma block) so we round down to TX_32X32.
if (subsampled && max_txsize == TX_64X64) max_txsize = TX_32X32;
#else
(void)subsampled;
#endif
return max_txsize;
}
#endif // CONFIG_VAR_TX
......
......@@ -1406,7 +1406,7 @@ static const TX_SIZE uv_txsize_lookup[BLOCK_SIZES_ALL][TX_SIZES_ALL][2][2] = {
{ { TX_16X16, TX_16X16 }, { TX_16X16, TX_16X16 } },
{ { TX_32X32, TX_32X32 }, { TX_32X32, TX_32X32 } },
#if CONFIG_TX64X64
{ { TX_64X64, TX_64X64 }, { TX_32X32, TX_32X32 } },
{ { TX_64X64, TX_32X32 }, { TX_32X32, TX_32X32 } },
#endif // CONFIG_TX64X64
{ { TX_4X8, TX_4X8 }, { TX_4X8, TX_4X8 } },
{ { TX_8X4, TX_8X4 }, { TX_8X4, TX_8X4 } },
......@@ -1429,7 +1429,7 @@ static const TX_SIZE uv_txsize_lookup[BLOCK_SIZES_ALL][TX_SIZES_ALL][2][2] = {
{ { TX_16X16, TX_16X16 }, { TX_16X16, TX_16X16 } },
{ { TX_32X32, TX_32X32 }, { TX_32X32, TX_32X32 } },
#if CONFIG_TX64X64
{ { TX_64X64, TX_32X32 }, { TX_64X64, TX_32X32 } },
{ { TX_64X64, TX_32X32 }, { TX_32X32, TX_32X32 } },
#endif // CONFIG_TX64X64
{ { TX_4X8, TX_4X8 }, { TX_4X8, TX_4X8 } },
{ { TX_8X4, TX_8X4 }, { TX_8X4, TX_8X4 } },
......@@ -1452,7 +1452,7 @@ static const TX_SIZE uv_txsize_lookup[BLOCK_SIZES_ALL][TX_SIZES_ALL][2][2] = {
{ { TX_16X16, TX_16X16 }, { TX_16X16, TX_16X16 } },
{ { TX_32X32, TX_32X32 }, { TX_32X32, TX_32X32 } },
#if CONFIG_TX64X64
{ { TX_64X64, TX_64X64 }, { TX_64X64, TX_64X64 } },
{ { TX_64X64, TX_32X32 }, { TX_32X32, TX_32X32 } },
#endif // CONFIG_TX64X64
{ { TX_4X8, TX_4X8 }, { TX_4X8, TX_4X8 } },
{ { TX_8X4, TX_8X4 }, { TX_8X4, TX_8X4 } },
......
......@@ -2068,7 +2068,8 @@ static void decode_token_and_recon_block(AV1Decoder *const pbi,
mu_blocks_wide = AOMMIN(max_blocks_wide, mu_blocks_wide);
mu_blocks_high = AOMMIN(max_blocks_high, mu_blocks_high);
const TX_SIZE max_tx_size = get_vartx_max_txsize(mbmi, plane_bsize);
const TX_SIZE max_tx_size = get_vartx_max_txsize(
mbmi, plane_bsize, pd->subsampling_x || pd->subsampling_y);
const int bh_var_tx = tx_size_high_unit[max_tx_size];
const int bw_var_tx = tx_size_wide_unit[max_tx_size];
int block = 0;
......
......@@ -1761,7 +1761,7 @@ static void pack_inter_mode_mvs(AV1_COMP *cpi, const int mi_row,
!(is_inter && skip) && !xd->lossless[segment_id]) {
#if CONFIG_VAR_TX
if (is_inter) { // This implies skip flag is 0.
const TX_SIZE max_tx_size = get_vartx_max_txsize(mbmi, bsize);
const TX_SIZE max_tx_size = get_vartx_max_txsize(mbmi, bsize, 0);
const int bh = tx_size_high_unit[max_tx_size];
const int bw = tx_size_wide_unit[max_tx_size];
const int width = block_size_wide[bsize] >> tx_size_wide_log2[0];
......@@ -2630,12 +2630,15 @@ static void write_tokens_b(AV1_COMP *cpi, const TileInfo *const tile,
mu_blocks_high = AOMMIN(num_4x4_h, mu_blocks_high);
if (is_inter_block(mbmi)) {
const TX_SIZE max_tx_size = get_vartx_max_txsize(mbmi, plane_bsize);
const TX_SIZE max_tx_size = get_vartx_max_txsize(
mbmi, plane_bsize, pd->subsampling_x || pd->subsampling_y);
int block = 0;
const int step =
tx_size_wide_unit[max_tx_size] * tx_size_high_unit[max_tx_size];
const int bkw = tx_size_wide_unit[max_tx_size];
const int bkh = tx_size_high_unit[max_tx_size];
assert(bkw <= mu_blocks_wide);
assert(bkh <= mu_blocks_high);
for (row = 0; row < num_4x4_h; row += mu_blocks_high) {
const int unit_height = AOMMIN(mu_blocks_high + row, num_4x4_h);
for (col = 0; col < num_4x4_w; col += mu_blocks_wide) {
......
......@@ -5927,7 +5927,7 @@ static void tx_partition_count_update(const AV1_COMMON *const cm, MACROBLOCK *x,
MACROBLOCKD *xd = &x->e_mbd;
const int mi_width = block_size_wide[plane_bsize] >> tx_size_wide_log2[0];
const int mi_height = block_size_high[plane_bsize] >> tx_size_wide_log2[0];
TX_SIZE max_tx_size = get_vartx_max_txsize(&xd->mi[0]->mbmi, plane_bsize);
TX_SIZE max_tx_size = get_vartx_max_txsize(&xd->mi[0]->mbmi, plane_bsize, 0);
const int bh = tx_size_high_unit[max_tx_size];
const int bw = tx_size_wide_unit[max_tx_size];
int idx, idy;
......@@ -5986,7 +5986,7 @@ static void tx_partition_set_contexts(const AV1_COMMON *const cm,
int mi_row, int mi_col) {
const int mi_width = block_size_wide[plane_bsize] >> tx_size_wide_log2[0];
const int mi_height = block_size_high[plane_bsize] >> tx_size_high_log2[0];
TX_SIZE max_tx_size = get_vartx_max_txsize(&xd->mi[0]->mbmi, plane_bsize);
TX_SIZE max_tx_size = get_vartx_max_txsize(&xd->mi[0]->mbmi, plane_bsize, 0);
const int bh = tx_size_high_unit[max_tx_size];
const int bw = tx_size_wide_unit[max_tx_size];
int idx, idy;
......
......@@ -942,7 +942,8 @@ void av1_encode_sb(AV1_COMMON *cm, MACROBLOCK *x, BLOCK_SIZE bsize, int mi_row,
const BLOCK_SIZE plane_bsize = get_plane_block_size(bsize, pd);
const int mi_width = block_size_wide[plane_bsize] >> tx_size_wide_log2[0];
const int mi_height = block_size_high[plane_bsize] >> tx_size_wide_log2[0];
const TX_SIZE max_tx_size = get_vartx_max_txsize(mbmi, plane_bsize);
const TX_SIZE max_tx_size = get_vartx_max_txsize(
mbmi, plane_bsize, pd->subsampling_x || pd->subsampling_y);
const BLOCK_SIZE txb_size = txsize_to_bsize[max_tx_size];
const int bw = block_size_wide[txb_size] >> tx_size_wide_log2[0];
const int bh = block_size_high[txb_size] >> tx_size_wide_log2[0];
......
......@@ -757,7 +757,8 @@ void av1_tokenize_sb_vartx(const AV1_COMP *cpi, ThreadData *td, TOKENEXTRA **t,
#endif
const int mi_width = block_size_wide[plane_bsize] >> tx_size_wide_log2[0];
const int mi_height = block_size_high[plane_bsize] >> tx_size_wide_log2[0];
const TX_SIZE max_tx_size = get_vartx_max_txsize(mbmi, plane_bsize);
const TX_SIZE max_tx_size = get_vartx_max_txsize(
mbmi, plane_bsize, pd->subsampling_x || pd->subsampling_y);
const BLOCK_SIZE txb_size = txsize_to_bsize[max_tx_size];
int bw = block_size_wide[txb_size] >> tx_size_wide_log2[0];
int bh = block_size_high[txb_size] >> tx_size_wide_log2[0];
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment