Commit e7c0e157 authored by Geza Lore's avatar Geza Lore

Set inter_tx_size for supertx coded blocks.

The loop filter relies on inter_tx_size in MB_MODE_INFO being set
properly when VAR_TX is enabled. Supertx coded blocks did not set this
previously at all, and the differing garbage values eventually resulted
in in a YUV mismatch between encoder and decoder after loop filtering.

This patch fixes this by setting inter_tx_size to the proper supertx
size in both the encoder and the decoder. This should also mean that
loop filtering is done at the proper transform boundaries, even when
supertx or vartx is being used.

Change-Id: I41a564cd6d34ce4a8313ad4efa89d905f5ead731
parent a8122bb9
......@@ -281,6 +281,9 @@ typedef struct macroblockd {
TXFM_CONTEXT left_txfm_context_buffer[8];
TX_SIZE max_tx_size;
#if CONFIG_SUPERTX
TX_SIZE supertx_size;
#endif
#endif
// dimension in the unit of 8x8 block of the current block
......
......@@ -1829,6 +1829,9 @@ static void decode_partition(VP10Decoder *const pbi, MACROBLOCKD *const xd,
}
#endif // CONFIG_EXT_TX
}
#if CONFIG_VAR_TX
xd->supertx_size = supertx_size;
#endif
}
#endif // CONFIG_SUPERTX
if (!hbs) {
......
......@@ -1283,6 +1283,17 @@ static void read_inter_frame_mode_info(VP10Decoder *const pbi,
#endif // CONFIG_VAR_TX
#if CONFIG_SUPERTX
}
#if CONFIG_VAR_TX
else if (inter_block) {
const int width = num_4x4_blocks_wide_lookup[bsize];
const int height = num_4x4_blocks_high_lookup[bsize];
int idx, idy;
xd->mi[0]->mbmi.tx_size = xd->supertx_size;
for (idy = 0; idy < height; ++idy)
for (idx = 0; idx < width; ++idx)
xd->mi[0]->mbmi.inter_tx_size[(idy >> 1) * 8 + (idx >> 1)] = xd->supertx_size;
}
#endif // CONFIG_VAR_TX
#endif // CONFIG_SUPERTX
if (inter_block)
......
......@@ -1236,6 +1236,7 @@ static void update_state_supertx(VP10_COMP *cpi, ThreadData *td,
*mi_addr = *mi;
*x->mbmi_ext = ctx->mbmi_ext;
assert(is_inter_block(mbmi));
assert(mbmi->tx_size == ctx->mic.mbmi.tx_size);
// If segmentation in use
if (seg->enabled && output_enabled) {
......@@ -1309,6 +1310,16 @@ static void update_state_supertx(VP10_COMP *cpi, ThreadData *td,
mv->mv[1].as_int = mi->mbmi.mv[1].as_int;
}
}
#if CONFIG_VAR_TX
{
const TX_SIZE mtx = mbmi->tx_size;
int idy, idx;
for (idy = 0; idy < (1 << mtx) / 2; ++idy)
for (idx = 0; idx < (1 << mtx) / 2; ++idx)
mbmi->inter_tx_size[(idy << 3) + idx] = mbmi->tx_size;
}
#endif
}
static void update_state_sb_supertx(VP10_COMP *cpi, ThreadData *td,
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment