Commit 36661926 authored by Sarah Parker's avatar Sarah Parker

Avoid exiting tx size search when tx-size is square

This fixes a mismatch in ext-tx + rect-tx introduced
by a refactor in 2d147c16.

BUG=aomedia:422

Change-Id: I5bc40b2dcd7bf53b20c1bea454aae49f2d5dfe67
parent 27acc478
......@@ -1938,9 +1938,6 @@ static int skip_txfm_search(const AV1_COMP *cpi, MACROBLOCK *x, BLOCK_SIZE bs,
#if CONFIG_REF_MV
if (mbmi->ref_mv_idx > 0 && tx_type != DCT_DCT) return 1;
#endif // CONFIG_REF_MV
#if CONFIG_EXT_TX && CONFIG_RECT_TX
if (!is_rect_tx(tx_size)) return 1;
#endif // CONFIG_EXT_TX && CONFIG_RECT_TX
if (FIXED_TX_TYPE && tx_type != get_default_tx_type(0, xd, 0, tx_size))
return 1;
if (!is_inter && x->use_default_intra_tx_type &&
......@@ -2266,6 +2263,9 @@ static void choose_tx_size_type_from_rd(const AV1_COMP *const cpi,
last_rd = INT64_MAX;
for (n = start_tx; n >= end_tx; --n) {
#if CONFIG_EXT_TX && CONFIG_RECT_TX
if (is_rect_tx(n)) break;
#endif // CONFIG_EXT_TX && CONFIG_RECT_TX
TX_TYPE tx_start = DCT_DCT;
TX_TYPE tx_end = TX_TYPES;
#if CONFIG_LV_MAP
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment