From e16b2665c29c4ba9e5da8b264135b6a40f2c1cb5 Mon Sep 17 00:00:00 2001 From: Alex Converse Date: Wed, 24 May 2017 14:00:00 -0700 Subject: [PATCH] var_tx+intrabc: Fix infinite recursion Change-Id: Ibbf0c14934d0bff316cfdf5c252c64b3e68c73e7 --- av1/decoder/decodeframe.c | 1 + av1/decoder/decodemv.c | 10 ++++++++++ av1/encoder/encodemb.c | 3 ++- av1/encoder/rdopt.c | 11 +++++++++++ 4 files changed, 24 insertions(+), 1 deletion(-) diff --git a/av1/decoder/decodeframe.c b/av1/decoder/decodeframe.c index bdf38540e..e377970d4 100644 --- a/av1/decoder/decodeframe.c +++ b/av1/decoder/decodeframe.c @@ -753,6 +753,7 @@ static void decode_reconstruct_tx(AV1_COMMON *cm, MACROBLOCKD *const xd, } else { const TX_SIZE sub_txs = sub_tx_size_map[tx_size]; const int bsl = tx_size_wide_unit[sub_txs]; + assert(sub_txs < tx_size); int i; assert(bsl > 0); diff --git a/av1/decoder/decodemv.c b/av1/decoder/decodemv.c index d8cc52523..35ebc6c65 100644 --- a/av1/decoder/decodemv.c +++ b/av1/decoder/decodemv.c @@ -1121,6 +1121,16 @@ static void read_intra_frame_mode_info(AV1_COMMON *const cm, xd->corrupted |= !assign_dv(cm, xd, &mbmi->mv[0], &dv_ref, mi_row, mi_col, bsize, r); +#if CONFIG_VAR_TX + // TODO(aconverse@google.com): Evaluate allowing VAR TX on intrabc blocks + const int width = block_size_wide[bsize] >> tx_size_wide_log2[0]; + const int height = block_size_high[bsize] >> tx_size_high_log2[0]; + int idx, idy; + for (idy = 0; idy < height; ++idy) + for (idx = 0; idx < width; ++idx) + mbmi->inter_tx_size[idy >> 1][idx >> 1] = mbmi->tx_size; + mbmi->min_tx_size = get_min_tx_size(mbmi->tx_size); +#endif // CONFIG_VAR_TX #if CONFIG_EXT_TX && !CONFIG_TXK_SEL av1_read_tx_type(cm, xd, #if CONFIG_SUPERTX diff --git a/av1/encoder/encodemb.c b/av1/encoder/encodemb.c index af9d5d260..15012a92c 100644 --- a/av1/encoder/encodemb.c +++ b/av1/encoder/encodemb.c @@ -1188,12 +1188,13 @@ static void encode_block_inter(int plane, int block, int blk_row, int blk_col, if (tx_size == plane_tx_size) { encode_block(plane, block, blk_row, blk_col, plane_bsize, tx_size, arg); } else { + assert(tx_size < TX_SIZES_ALL); const TX_SIZE sub_txs = sub_tx_size_map[tx_size]; + assert(sub_txs < tx_size); // This is the square transform block partition entry point. int bsl = tx_size_wide_unit[sub_txs]; int i; assert(bsl > 0); - assert(tx_size < TX_SIZES_ALL); for (i = 0; i < 4; ++i) { const int offsetr = blk_row + ((i >> 1) * bsl); diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c index 17c52a836..9c164eb8e 100644 --- a/av1/encoder/rdopt.c +++ b/av1/encoder/rdopt.c @@ -9417,6 +9417,17 @@ static int64_t rd_pick_intrabc_mode_sb(const AV1_COMP *cpi, MACROBLOCK *x, mbmi->rd_stats = rd_stats; #endif +#if CONFIG_VAR_TX + // TODO(aconverse@google.com): Evaluate allowing VAR TX on intrabc blocks + const int width = block_size_wide[bsize] >> tx_size_wide_log2[0]; + const int height = block_size_high[bsize] >> tx_size_high_log2[0]; + int idx, idy; + for (idy = 0; idy < height; ++idy) + for (idx = 0; idx < width; ++idx) + mbmi->inter_tx_size[idy >> 1][idx >> 1] = mbmi->tx_size; + mbmi->min_tx_size = get_min_tx_size(mbmi->tx_size); +#endif // CONFIG_VAR_TX + const aom_prob skip_prob = av1_get_skip_prob(cm, xd); RD_STATS rdc_noskip; -- GitLab