Commit 106b3cbc authored by Sarah Parker's avatar Sarah Parker

Use correct bsize in vartx sub8x8 max tx_size lookup

This fixes crashes due to infinite recursion when var_tx,
ext_tx and rect_tx are enabled without cb4x4. This
is the first part of an ongoing fix for this experiment
interaction.

Change-Id: I674f28294666102aff2265f6b6112816cac17378
parent 8e173429
......@@ -1147,6 +1147,18 @@ static INLINE int is_interintra_pred(const MB_MODE_INFO *mbmi) {
}
#endif // CONFIG_EXT_INTER
#if CONFIG_VAR_TX
static INLINE int get_vartx_max_txsize(const MB_MODE_INFO *const mbmi,
BLOCK_SIZE bsize) {
#if CONFIG_CB4X4
(void)mbmi;
return max_txsize_rect_lookup[bsize];
#endif // CONFIG_C4X4
return mbmi->sb_type < BLOCK_8X8 ? max_txsize_rect_lookup[mbmi->sb_type]
: max_txsize_rect_lookup[bsize];
}
#endif // CONFIG_VAR_TX
#if CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION
static INLINE int is_motion_variation_allowed_bsize(BLOCK_SIZE bsize) {
return (bsize >= BLOCK_8X8);
......
......@@ -1794,7 +1794,7 @@ static void decode_token_and_recon_block(AV1Decoder *const pbi,
#endif
#if CONFIG_VAR_TX
const TX_SIZE max_tx_size = max_txsize_rect_lookup[plane_bsize];
const TX_SIZE max_tx_size = get_vartx_max_txsize(mbmi, plane_bsize);
const int bh_var_tx = tx_size_high_unit[max_tx_size];
const int bw_var_tx = tx_size_wide_unit[max_tx_size];
for (row = 0; row < max_blocks_high; row += bh_var_tx)
......
......@@ -1742,7 +1742,7 @@ static void pack_inter_mode_mvs(AV1_COMP *cpi, const int mi_row,
!(is_inter && skip) && !xd->lossless[segment_id]) {
#if CONFIG_VAR_TX
if (is_inter) { // This implies skip flag is 0.
const TX_SIZE max_tx_size = max_txsize_rect_lookup[bsize];
const TX_SIZE max_tx_size = get_vartx_max_txsize(mbmi, bsize);
const int bh = tx_size_high_unit[max_tx_size];
const int bw = tx_size_wide_unit[max_tx_size];
const int width = block_size_wide[bsize] >> tx_size_wide_log2[0];
......@@ -2442,7 +2442,7 @@ static void write_tokens_b(AV1_COMP *cpi, const TileInfo *const tile,
init_token_stats(&token_stats);
if (is_inter_block(mbmi)) {
const TX_SIZE max_tx_size = max_txsize_rect_lookup[plane_bsize];
const TX_SIZE max_tx_size = get_vartx_max_txsize(mbmi, plane_bsize);
int block = 0;
const int step =
tx_size_wide_unit[max_tx_size] * tx_size_high_unit[max_tx_size];
......
......@@ -5743,7 +5743,7 @@ static void tx_partition_count_update(const AV1_COMMON *const cm, MACROBLOCK *x,
MACROBLOCKD *xd = &x->e_mbd;
const int mi_width = block_size_wide[plane_bsize] >> tx_size_wide_log2[0];
const int mi_height = block_size_high[plane_bsize] >> tx_size_wide_log2[0];
TX_SIZE max_tx_size = max_txsize_rect_lookup[plane_bsize];
TX_SIZE max_tx_size = get_vartx_max_txsize(&xd->mi[0]->mbmi, plane_bsize);
const int bh = tx_size_high_unit[max_tx_size];
const int bw = tx_size_wide_unit[max_tx_size];
int idx, idy;
......@@ -5801,7 +5801,7 @@ static void tx_partition_set_contexts(const AV1_COMMON *const cm,
int mi_row, int mi_col) {
const int mi_width = block_size_wide[plane_bsize] >> tx_size_wide_log2[0];
const int mi_height = block_size_high[plane_bsize] >> tx_size_high_log2[0];
TX_SIZE max_tx_size = max_txsize_rect_lookup[plane_bsize];
TX_SIZE max_tx_size = get_vartx_max_txsize(&xd->mi[0]->mbmi, plane_bsize);
const int bh = tx_size_high_unit[max_tx_size];
const int bw = tx_size_wide_unit[max_tx_size];
int idx, idy;
......
......@@ -1320,7 +1320,7 @@ void av1_encode_sb(AV1_COMMON *cm, MACROBLOCK *x, BLOCK_SIZE bsize,
const BLOCK_SIZE plane_bsize = get_plane_block_size(bsize, pd);
const int mi_width = block_size_wide[plane_bsize] >> tx_size_wide_log2[0];
const int mi_height = block_size_high[plane_bsize] >> tx_size_wide_log2[0];
const TX_SIZE max_tx_size = max_txsize_rect_lookup[plane_bsize];
const TX_SIZE max_tx_size = get_vartx_max_txsize(mbmi, plane_bsize);
const BLOCK_SIZE txb_size = txsize_to_bsize[max_tx_size];
const int bw = block_size_wide[txb_size] >> tx_size_wide_log2[0];
const int bh = block_size_high[txb_size] >> tx_size_wide_log2[0];
......
......@@ -731,7 +731,7 @@ void av1_tokenize_sb_vartx(const AV1_COMP *cpi, ThreadData *td, TOKENEXTRA **t,
#endif
const int mi_width = block_size_wide[plane_bsize] >> tx_size_wide_log2[0];
const int mi_height = block_size_high[plane_bsize] >> tx_size_wide_log2[0];
const TX_SIZE max_tx_size = max_txsize_rect_lookup[plane_bsize];
const TX_SIZE max_tx_size = get_vartx_max_txsize(mbmi, plane_bsize);
const BLOCK_SIZE txb_size = txsize_to_bsize[max_tx_size];
int bw = block_size_wide[txb_size] >> tx_size_wide_log2[0];
int bh = block_size_high[txb_size] >> tx_size_wide_log2[0];
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment