Commit 49587a77 authored by Yue Chen's avatar Yue Chen Committed by Debargha Mukherjee

Fix unit test failure for RECT_TX + VAR_TX

Disable rect_tx because we only support 4x4 Walsh-Hadamard transform
in lossless mode.

Fixes failure in ./test_libaom --gtest_filter=*Large*ScreencastQ0/1
Configuration: --enable-experimental --enable-var-tx --enable-rect-tx
 --enable-ref-mv --enable-ext_intra --enable-ext_tx --enable-debug
 --disable-optimizations

Change-Id: Ib6b3494c7dcf7182f1cab9b138388d054851a23d
parent 6c6ddac3
......@@ -516,8 +516,10 @@ static INLINE int is_rect_tx_allowed_bsize(BLOCK_SIZE bsize) {
return LUT[bsize];
}
static INLINE int is_rect_tx_allowed(const MB_MODE_INFO *mbmi) {
return is_inter_block(mbmi) && is_rect_tx_allowed_bsize(mbmi->sb_type);
static INLINE int is_rect_tx_allowed(const MACROBLOCKD *xd,
const MB_MODE_INFO *mbmi) {
return is_inter_block(mbmi) && is_rect_tx_allowed_bsize(mbmi->sb_type) &&
!xd->lossless[mbmi->segment_id];
}
static INLINE int is_rect_tx(TX_SIZE tx_size) { return tx_size >= TX_SIZES; }
......
......@@ -1633,7 +1633,8 @@ static void read_inter_frame_mode_info(AV1Decoder *const pbi,
int idx, idy;
int tx_size_cat = inter_tx_size_cat_lookup[bsize];
#if CONFIG_EXT_TX && CONFIG_RECT_TX
int is_rect_tx_allowed = inter_block && is_rect_tx_allowed_bsize(bsize);
int is_rect_tx_allowed = inter_block && is_rect_tx_allowed_bsize(bsize) &&
!xd->lossless[mbmi->segment_id];
int use_rect_tx = 0;
if (is_rect_tx_allowed) {
......
......@@ -406,7 +406,7 @@ static void write_selected_tx_size(const AV1_COMMON *cm, const MACROBLOCKD *xd,
const TX_SIZE coded_tx_size = txsize_sqr_up_map[tx_size];
#if CONFIG_EXT_TX && CONFIG_RECT_TX
assert(IMPLIES(is_rect_tx(tx_size), is_rect_tx_allowed(mbmi)));
assert(IMPLIES(is_rect_tx(tx_size), is_rect_tx_allowed(xd, mbmi)));
assert(
IMPLIES(is_rect_tx(tx_size), tx_size == max_txsize_rect_lookup[bsize]));
#endif // CONFIG_EXT_TX && CONFIG_RECT_TX
......@@ -1132,7 +1132,7 @@ static void pack_inter_mode_mvs(AV1_COMP *cpi, const MODE_INFO *mi,
int idx, idy;
#if CONFIG_EXT_TX && CONFIG_RECT_TX
if (is_rect_tx_allowed(mbmi)) {
if (is_rect_tx_allowed(xd, mbmi)) {
int tx_size_cat = inter_tx_size_cat_lookup[bsize];
aom_write(w, is_rect_tx(mbmi->tx_size),
......
......@@ -5141,14 +5141,14 @@ static void encode_superblock(AV1_COMP *cpi, ThreadData *td, TOKENEXTRA **t,
: intra_tx_size_cat_lookup[bsize];
const TX_SIZE coded_tx_size = txsize_sqr_up_map[mbmi->tx_size];
#if CONFIG_EXT_TX && CONFIG_RECT_TX
assert(IMPLIES(is_rect_tx(mbmi->tx_size), is_rect_tx_allowed(mbmi)));
assert(IMPLIES(is_rect_tx(mbmi->tx_size), is_rect_tx_allowed(xd, mbmi)));
#endif // CONFIG_EXT_TX && CONFIG_RECT_TX
#if CONFIG_VAR_TX
#if CONFIG_EXT_TX && CONFIG_RECT_TX
if (is_rect_tx_allowed(mbmi)) {
if (is_rect_tx_allowed(xd, mbmi)) {
td->counts->rect_tx[tx_size_cat][is_rect_tx(mbmi->tx_size)]++;
}
if (!is_rect_tx_allowed(mbmi) || !is_rect_tx(mbmi->tx_size)) {
if (!is_rect_tx_allowed(xd, mbmi) || !is_rect_tx(mbmi->tx_size)) {
#endif // CONFIG_EXT_TX && CONFIG_RECT_TX
if (is_inter)
tx_partition_count_update(cm, xd, bsize, mi_row, mi_col, td->counts);
......
......@@ -1385,14 +1385,14 @@ static int64_t choose_tx_size_fix_type(AV1_COMP *cpi, BLOCK_SIZE bs,
const int is_inter = is_inter_block(mbmi);
#if CONFIG_EXT_TX
#if CONFIG_RECT_TX
int evaulate_rect_tx = 0;
int evaluate_rect_tx = 0;
#endif // CONFIG_RECT_TX
int ext_tx_set;
#endif // CONFIG_EXT_TX
if (tx_select) {
#if CONFIG_EXT_TX && CONFIG_RECT_TX
evaulate_rect_tx = is_rect_tx_allowed(mbmi);
evaluate_rect_tx = is_rect_tx_allowed(xd, mbmi);
#endif // CONFIG_EXT_TX && CONFIG_RECT_TX
start_tx = max_tx_size;
end_tx = 0;
......@@ -1400,8 +1400,8 @@ static int64_t choose_tx_size_fix_type(AV1_COMP *cpi, BLOCK_SIZE bs,
const TX_SIZE chosen_tx_size =
tx_size_from_tx_mode(bs, cm->tx_mode, is_inter);
#if CONFIG_EXT_TX && CONFIG_RECT_TX
evaulate_rect_tx = is_rect_tx(chosen_tx_size);
assert(IMPLIES(evaulate_rect_tx, is_rect_tx_allowed(mbmi)));
evaluate_rect_tx = is_rect_tx(chosen_tx_size);
assert(IMPLIES(evaluate_rect_tx, is_rect_tx_allowed(xd, mbmi)));
#endif // CONFIG_EXT_TX && CONFIG_RECT_TX
start_tx = chosen_tx_size;
end_tx = chosen_tx_size;
......@@ -1415,7 +1415,7 @@ static int64_t choose_tx_size_fix_type(AV1_COMP *cpi, BLOCK_SIZE bs,
mbmi->tx_type = tx_type;
#if CONFIG_EXT_TX && CONFIG_RECT_TX
if (evaulate_rect_tx) {
if (evaluate_rect_tx) {
const TX_SIZE rect_tx_size = max_txsize_rect_lookup[bs];
const int ext_tx_set = get_ext_tx_set(rect_tx_size, bs, 1);
if (ext_tx_used_inter[ext_tx_set][tx_type]) {
......@@ -3214,7 +3214,7 @@ static int64_t select_tx_size_fix_type(const AV1_COMP *cpi, MACROBLOCK *x,
mbmi->tx_type = tx_type;
inter_block_yrd(cpi, x, rate, dist, skippable, sse, bsize, ref_best_rd);
#if CONFIG_EXT_TX && CONFIG_RECT_TX
if (is_rect_tx_allowed(mbmi)) {
if (is_rect_tx_allowed(xd, mbmi)) {
int rate_rect_tx, skippable_rect_tx = 0;
int64_t dist_rect_tx, sse_rect_tx, rd, rd_rect_tx;
int tx_size_cat = inter_tx_size_cat_lookup[bsize];
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment