Commit 07ef967d authored by Jingning Han's avatar Jingning Han

Resolve coding failure in var-tx

Fix an encoding failure issue when var-tx is enabled, while ext-tx
and rect-tx are disabled. This doesn't change coding statistics
when all are enabled.

Change-Id: I4b32387a0a1497380980f8087832aaf6467cdcbe
parent 3daa4fda
......@@ -487,7 +487,7 @@ static const TX_SIZE max_txsize_rect_lookup[BLOCK_SIZES] = {
#endif // CONFIG_TX64X64
};
#if CONFIG_EXT_TX && CONFIG_RECT_TX
#if (CONFIG_VAR_TX || CONFIG_RECT_TX)
// Same as "max_txsize_lookup[bsize] - TX_8X8", except for rectangular
// block which may use a rectangular transform, in which case it is
// "(max_txsize_lookup[bsize] + 1) - TX_8X8", invalid for bsize < 8X8
......@@ -560,7 +560,7 @@ static const int32_t intra_tx_size_cat_lookup[BLOCK_SIZES] = {
#endif // CONFIG_EXT_PARTITION
#endif // CONFIG_TX64X64
};
#endif // CONFIG_EXT_TX && CONFIG_RECT_TX
#endif // CONFIG_VAR_TX && CONFIG_RECT_TX
#define inter_tx_size_cat_lookup intra_tx_size_cat_lookup
......
......@@ -1973,7 +1973,7 @@ void av1_predict_intra_block(const MACROBLOCKD *xd, int wpx, int hpx,
predict_square_intra_block(xd, wpx, hpx, tx_size, mode, ref, ref_stride,
dst, dst_stride, col_off, row_off, plane);
} else {
#if CONFIG_EXT_TX && CONFIG_RECT_TX
#if (CONFIG_VAR_TX || CONFIG_RECT_TX)
#if CONFIG_AOM_HIGHBITDEPTH
uint16_t tmp16[MAX_SB_SIZE];
#endif
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment