Commit c7fa12c0 authored by Sarah Parker's avatar Sarah Parker

Avoid using ADST for tx sizes > 16x16

When the reduced transform set is enabled in the frame level flag,
it is possible to select ADST for large tx sizes. This patch
corrects this issue.

BUG=aomedia:1280

Change-Id: I3c6166d0258cbf3179d74a911e420cea85e5299c
parent 39216d7f
......@@ -763,10 +763,10 @@ static INLINE TxSetType get_ext_tx_set_type(TX_SIZE tx_size, BLOCK_SIZE bs,
#else
if (tx_size_sqr_up > TX_32X32 || bs < BLOCK_8X8) return EXT_TX_SET_DCTONLY;
#endif // USE_TXTYPE_SEARCH_FOR_SUB8X8_IN_CB4X4
if (use_reduced_set)
return is_inter ? EXT_TX_SET_DCT_IDTX : EXT_TX_SET_DTT4_IDTX;
if (tx_size_sqr_up == TX_32X32)
return is_inter ? EXT_TX_SET_DCT_IDTX : EXT_TX_SET_DCTONLY;
if (use_reduced_set)
return is_inter ? EXT_TX_SET_DCT_IDTX : EXT_TX_SET_DTT4_IDTX;
if (is_inter) {
return (tx_size_sqr == TX_16X16 ? EXT_TX_SET_DTT9_IDTX_1DDCT
: EXT_TX_SET_ALL16);
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment