Commit 3ca7dd98 authored by Yue Chen's avatar Yue Chen

Fix assertion failure in var_tx+rect_tx-ext_tx

BUG=aomedia:560

Change-Id: Ie399d61162ab33e35fa0cb04206904e3af1bbb2d
parent f19f35f7
...@@ -507,10 +507,10 @@ static TX_SIZE read_tx_size(AV1_COMMON *cm, MACROBLOCKD *xd, int is_inter, ...@@ -507,10 +507,10 @@ static TX_SIZE read_tx_size(AV1_COMMON *cm, MACROBLOCKD *xd, int is_inter,
: intra_tx_size_cat_lookup[bsize]; : intra_tx_size_cat_lookup[bsize];
const TX_SIZE coded_tx_size = const TX_SIZE coded_tx_size =
read_selected_tx_size(cm, xd, tx_size_cat, r); read_selected_tx_size(cm, xd, tx_size_cat, r);
#if CONFIG_EXT_TX && CONFIG_RECT_TX #if CONFIG_RECT_TX && (CONFIG_EXT_TX || CONFIG_VAR_TX)
if (coded_tx_size > max_txsize_lookup[bsize]) { if (coded_tx_size > max_txsize_lookup[bsize]) {
assert(coded_tx_size == max_txsize_lookup[bsize] + 1); assert(coded_tx_size == max_txsize_lookup[bsize] + 1);
#if CONFIG_RECT_TX_EXT #if CONFIG_EXT_TX && CONFIG_RECT_TX_EXT
if (is_quarter_tx_allowed(xd, &xd->mi[0]->mbmi, is_inter)) { if (is_quarter_tx_allowed(xd, &xd->mi[0]->mbmi, is_inter)) {
int quarter_tx = aom_read(r, cm->fc->quarter_tx_size_prob, ACCT_STR); int quarter_tx = aom_read(r, cm->fc->quarter_tx_size_prob, ACCT_STR);
FRAME_COUNTS *counts = xd->counts; FRAME_COUNTS *counts = xd->counts;
...@@ -519,13 +519,13 @@ static TX_SIZE read_tx_size(AV1_COMMON *cm, MACROBLOCKD *xd, int is_inter, ...@@ -519,13 +519,13 @@ static TX_SIZE read_tx_size(AV1_COMMON *cm, MACROBLOCKD *xd, int is_inter,
return quarter_tx ? quarter_txsize_lookup[bsize] return quarter_tx ? quarter_txsize_lookup[bsize]
: max_txsize_rect_lookup[bsize]; : max_txsize_rect_lookup[bsize];
} }
#endif // CONFIG_RECT_TX_EXT #endif // CONFIG_EXT_TX && CONFIG_RECT_TX_EXT
return max_txsize_rect_lookup[bsize]; return max_txsize_rect_lookup[bsize];
} }
#else #else
assert(coded_tx_size <= max_txsize_lookup[bsize]); assert(coded_tx_size <= max_txsize_lookup[bsize]);
#endif // CONFIG_EXT_TX && CONFIG_RECT_TX #endif // CONFIG_RECT_TX && (CONFIG_EXT_TX || CONFIG_VAR_TX)
return coded_tx_size; return coded_tx_size;
} else { } else {
return tx_size_from_tx_mode(bsize, tx_mode, is_inter); return tx_size_from_tx_mode(bsize, tx_mode, is_inter);
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment