Commit 28d924b7 authored by Debargha Mukherjee's avatar Debargha Mukherjee

A bug fix for var-tx

Fixes a crash with supertx, ext-tx and rect-tx

Change-Id: I6b5f4cfd6e209558541a791be685b55156aa0138
parent cb603790
......@@ -1304,10 +1304,13 @@ static void update_state_supertx(AV1_COMP *cpi, ThreadData *td,
#if CONFIG_VAR_TX
{
const TX_SIZE mtx = mbmi->tx_size;
const int num_4x4_blocks_wide = num_4x4_blocks_wide_txsize_lookup[mtx] >> 1;
const int num_4x4_blocks_high = num_4x4_blocks_high_txsize_lookup[mtx] >> 1;
int idy, idx;
for (idy = 0; idy < (1 << mtx) / 2; ++idy)
for (idx = 0; idx < (1 << mtx) / 2; ++idx)
mbmi->inter_tx_size[idy][idx] = mbmi->tx_size;
mbmi->inter_tx_size[0][0] = mtx;
for (idy = 0; idy < num_4x4_blocks_high; ++idy)
for (idx = 0; idx < num_4x4_blocks_wide; ++idx)
mbmi->inter_tx_size[idy][idx] = mtx;
}
#endif // CONFIG_VAR_TX
// Turn motion variation off for supertx
......
......@@ -3321,7 +3321,7 @@ static void select_tx_type_yrd(const AV1_COMP *cpi, MACROBLOCK *x, int *rate,
TX_TYPE tx_type, best_tx_type = DCT_DCT;
const int is_inter = is_inter_block(mbmi);
TX_SIZE best_tx_size[MAX_MIB_SIZE][MAX_MIB_SIZE];
TX_SIZE best_tx = TX_SIZES;
TX_SIZE best_tx = max_txsize_lookup[bsize];
uint8_t best_blk_skip[MAX_MIB_SIZE * MAX_MIB_SIZE * 4];
const int n4 = 1 << (num_pels_log2_lookup[bsize] - 4);
int idx, idy;
......@@ -7707,6 +7707,7 @@ static int64_t handle_inter_mode(
} else {
x->skip = 1;
*disable_skip = 1;
mbmi->tx_size = tx_size_from_tx_mode(bsize, cm->tx_mode, 1);
// The cost of skip bit needs to be added.
#if CONFIG_OBMC || CONFIG_WARPED_MOTION
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment