Commit e4e18fcc authored by Debargha Mukherjee's avatar Debargha Mukherjee

Enable 4:1 -> 2:1 -> 1:1 splits for 4:1 transforms

Also, splits the sub_tx_size_map array into inter and intra
in order to enable the new 4:1 transforms for inter and
intra separately.
Includes refactoring such as removing the intra_tx_size_cat_lookup
array since it is unnecessary, and consolidating the
max_txsize_rect_lookup array for convenience.

Change-Id: I112553bab612dafb973611c87f36a43e1ac4be85
parent 4d8c6fc2
......@@ -944,10 +944,7 @@ static INLINE int is_rect_tx_allowed(const MACROBLOCKD *xd,
}
static INLINE TX_SIZE get_max_rect_tx_size(BLOCK_SIZE bsize, int is_inter) {
if (is_inter)
return max_txsize_rect_lookup[bsize];
else
return max_txsize_rect_intra_lookup[bsize];
return max_txsize_rect_lookup[is_inter][bsize];
}
static INLINE TX_SIZE tx_size_from_tx_mode(BLOCK_SIZE bsize, TX_MODE tx_mode,
......@@ -1102,18 +1099,31 @@ static INLINE int bsize_to_max_depth(BLOCK_SIZE bsize, int is_inter) {
int depth = 0;
while (depth < MAX_TX_DEPTH && tx_size != TX_4X4) {
depth++;
tx_size = sub_tx_size_map[tx_size];
tx_size = sub_tx_size_map[is_inter][tx_size];
}
return depth;
}
static INLINE int bsize_to_tx_size_cat(BLOCK_SIZE bsize, int is_inter) {
TX_SIZE tx_size = get_max_rect_tx_size(bsize, is_inter);
assert(tx_size != TX_4X4);
int depth = 0;
while (tx_size != TX_4X4) {
depth++;
tx_size = sub_tx_size_map[is_inter][tx_size];
assert(depth < 10);
}
assert(depth <= MAX_TX_CATS);
return depth - 1;
}
static INLINE int tx_size_to_depth(TX_SIZE tx_size, BLOCK_SIZE bsize,
int is_inter) {
TX_SIZE ctx_size = get_max_rect_tx_size(bsize, is_inter);
int depth = 0;
while (tx_size != ctx_size) {
depth++;
ctx_size = sub_tx_size_map[ctx_size];
ctx_size = sub_tx_size_map[is_inter][ctx_size];
assert(depth <= MAX_TX_DEPTH);
}
return depth;
......@@ -1123,7 +1133,7 @@ static INLINE TX_SIZE depth_to_tx_size(int depth, BLOCK_SIZE bsize,
int is_inter) {
TX_SIZE max_tx_size = get_max_rect_tx_size(bsize, is_inter);
TX_SIZE tx_size = max_tx_size;
for (int d = 0; d < depth; ++d) tx_size = sub_tx_size_map[tx_size];
for (int d = 0; d < depth; ++d) tx_size = sub_tx_size_map[is_inter][tx_size];
return tx_size;
}
......
This diff is collapsed.
......@@ -285,7 +285,7 @@ static void decode_reconstruct_tx(AV1_COMMON *cm, MACROBLOCKD *const xd,
pd->dst.stride, max_scan_line, eob, cm->reduced_tx_set_used);
*eob_total += eob;
} else {
const TX_SIZE sub_txs = sub_tx_size_map[tx_size];
const TX_SIZE sub_txs = sub_tx_size_map[1][tx_size];
assert(IMPLIES(tx_size <= TX_4X4, sub_txs == tx_size));
assert(IMPLIES(tx_size > TX_4X4, sub_txs < tx_size));
const int bsw = tx_size_wide_unit[sub_txs];
......@@ -532,7 +532,7 @@ static void decode_token_and_recon_block(AV1Decoder *const pbi,
const int is_split =
(l_max_tx_size != mbmi->inter_tx_size[0][0] && bsize == bsizec &&
txsize_to_bsize[l_max_tx_size] == bsizec);
if (is_split) max_tx_size = sub_tx_size_map[max_tx_size];
if (is_split) max_tx_size = sub_tx_size_map[1][max_tx_size];
}
#endif // DISABLE_VARTX_FOR_CHROMA == 2
const int bh_var_tx = tx_size_high_unit[max_tx_size];
......
......@@ -448,7 +448,7 @@ static void read_tx_size_vartx(AV1_COMMON *cm, MACROBLOCKD *xd,
is_split = aom_read_symbol(r, ec_ctx->txfm_partition_cdf[ctx], 2, ACCT_STR);
if (is_split) {
const TX_SIZE sub_txs = sub_tx_size_map[tx_size];
const TX_SIZE sub_txs = sub_tx_size_map[1][tx_size];
const int bsw = tx_size_wide_unit[sub_txs];
const int bsh = tx_size_high_unit[sub_txs];
......@@ -495,8 +495,7 @@ static TX_SIZE read_selected_tx_size(AV1_COMMON *cm, MACROBLOCKD *xd,
// TODO(debargha): Clean up the logic here. This function should only
// be called for intra.
const BLOCK_SIZE bsize = xd->mi[0]->mbmi.sb_type;
const int32_t tx_size_cat = is_inter ? inter_tx_size_cat_lookup[bsize]
: intra_tx_size_cat_lookup[bsize];
const int32_t tx_size_cat = bsize_to_tx_size_cat(bsize, is_inter);
const int max_depths = bsize_to_max_depth(bsize, 0);
const int ctx = get_tx_size_context(xd);
FRAME_CONTEXT *ec_ctx = xd->tile_ctx;
......@@ -957,7 +956,7 @@ void av1_read_tx_type(const AV1_COMMON *const cm, MACROBLOCKD *xd,
const TX_SIZE mtx_size =
get_max_rect_tx_size(xd->mi[0]->mbmi.sb_type, inter_block);
const TX_SIZE tx_size =
inter_block ? AOMMAX(sub_tx_size_map[mtx_size], mbmi->min_tx_size)
inter_block ? AOMMAX(sub_tx_size_map[1][mtx_size], mbmi->min_tx_size)
: mbmi->tx_size;
#endif // !CONFIG_TXK_SEL
FRAME_CONTEXT *ec_ctx = xd->tile_ctx;
......
......@@ -230,7 +230,7 @@ static void write_tx_size_vartx(const AV1_COMMON *cm, MACROBLOCKD *xd,
xd->left_txfm_context + blk_row, tx_size, tx_size);
// TODO(yuec): set correct txfm partition update for qttx
} else {
const TX_SIZE sub_txs = sub_tx_size_map[tx_size];
const TX_SIZE sub_txs = sub_tx_size_map[1][tx_size];
const int bsw = tx_size_wide_unit[sub_txs];
const int bsh = tx_size_high_unit[sub_txs];
......@@ -262,9 +262,9 @@ static void write_selected_tx_size(const AV1_COMMON *cm, const MACROBLOCKD *xd,
if (block_signals_txsize(bsize)) {
const TX_SIZE tx_size = mbmi->tx_size;
const int tx_size_ctx = get_tx_size_context(xd);
const int32_t tx_size_cat = intra_tx_size_cat_lookup[bsize];
const int depth = tx_size_to_depth(tx_size, bsize, 0);
const int max_depths = bsize_to_max_depth(bsize, 0);
const int32_t tx_size_cat = bsize_to_tx_size_cat(bsize, 0);
assert(depth >= 0 && depth <= max_depths);
assert(!is_inter_block(mbmi));
......@@ -536,7 +536,7 @@ static void pack_txb_tokens(aom_writer *w, AV1_COMMON *cm, MACROBLOCK *const x,
token_stats->cost += tmp_token_stats.cost;
#endif
} else {
const TX_SIZE sub_txs = sub_tx_size_map[tx_size];
const TX_SIZE sub_txs = sub_tx_size_map[1][tx_size];
const int bsw = tx_size_wide_unit[sub_txs];
const int bsh = tx_size_high_unit[sub_txs];
......@@ -592,7 +592,7 @@ static void pack_txb_tokens(aom_writer *w, const TOKENEXTRA **tp,
token_stats->cost += tmp_token_stats.cost;
#endif
} else {
const TX_SIZE sub_txs = sub_tx_size_map[tx_size];
const TX_SIZE sub_txs = sub_tx_size_map[1][tx_size];
const int bsw = tx_size_wide_unit[sub_txs];
const int bsh = tx_size_high_unit[sub_txs];
......@@ -1111,7 +1111,7 @@ void av1_write_tx_type(const AV1_COMMON *const cm, const MACROBLOCKD *xd,
const TX_SIZE mtx_size =
get_max_rect_tx_size(xd->mi[0]->mbmi.sb_type, is_inter);
const TX_SIZE tx_size =
is_inter ? AOMMAX(sub_tx_size_map[mtx_size], mbmi->min_tx_size)
is_inter ? AOMMAX(sub_tx_size_map[1][mtx_size], mbmi->min_tx_size)
: mbmi->tx_size;
#endif // !CONFIG_TXK_SEL
FRAME_CONTEXT *ec_ctx = xd->tile_ctx;
......@@ -1963,7 +1963,7 @@ static void write_inter_txb_coeff(AV1_COMMON *const cm, MACROBLOCK *const x,
const int is_split =
(l_max_tx_size != mbmi->inter_tx_size[0][0] && bsize == bsizec &&
txsize_to_bsize[l_max_tx_size] == bsizec);
if (is_split) max_tx_size = sub_tx_size_map[max_tx_size];
if (is_split) max_tx_size = sub_tx_size_map[1][max_tx_size];
}
#endif // DISABLE_VARTX_FOR_CHROMA == 2
const int step =
......
......@@ -4424,7 +4424,7 @@ static void sum_intra_stats(FRAME_COUNTS *counts, MACROBLOCKD *xd,
allow_update_cdf) {
const TX_SIZE tx_size = mbmi->tx_size;
const int tx_size_ctx = get_tx_size_context(xd);
const int32_t tx_size_cat = intra_tx_size_cat_lookup[bsize];
const int32_t tx_size_cat = bsize_to_tx_size_cat(bsize, 0);
const int depth = tx_size_to_depth(tx_size, bsize, 0);
const int max_depths = bsize_to_max_depth(bsize, 0);
update_cdf(fc->tx_size_cdf[tx_size_cat][tx_size_ctx], depth,
......@@ -4590,7 +4590,7 @@ static void update_txfm_count(MACROBLOCK *x, MACROBLOCKD *xd,
txfm_partition_update(xd->above_txfm_context + blk_col,
xd->left_txfm_context + blk_row, tx_size, tx_size);
} else {
const TX_SIZE sub_txs = sub_tx_size_map[tx_size];
const TX_SIZE sub_txs = sub_tx_size_map[1][tx_size];
const int bsw = tx_size_wide_unit[sub_txs];
const int bsh = tx_size_high_unit[sub_txs];
......@@ -4659,7 +4659,7 @@ static void set_txfm_context(MACROBLOCKD *xd, TX_SIZE tx_size, int blk_row,
xd->left_txfm_context + blk_row, tx_size, tx_size);
} else {
const TX_SIZE sub_txs = sub_tx_size_map[tx_size];
const TX_SIZE sub_txs = sub_tx_size_map[1][tx_size];
const int bsl = tx_size_wide_unit[sub_txs];
int i;
......
......@@ -685,7 +685,7 @@ static void encode_block_inter(int plane, int block, int blk_row, int blk_col,
encode_block(plane, block, blk_row, blk_col, plane_bsize, tx_size, arg);
} else {
assert(tx_size < TX_SIZES_ALL);
const TX_SIZE sub_txs = sub_tx_size_map[tx_size];
const TX_SIZE sub_txs = sub_tx_size_map[1][tx_size];
assert(IMPLIES(tx_size <= TX_4X4, sub_txs == tx_size));
assert(IMPLIES(tx_size > TX_4X4, sub_txs < tx_size));
// This is the square transform block partition entry point.
......@@ -807,7 +807,7 @@ void av1_encode_sb(AV1_COMMON *cm, MACROBLOCK *x, BLOCK_SIZE bsize, int mi_row,
const int is_split =
(l_max_tx_size != mbmi->inter_tx_size[0][0] && bsize == bsizec &&
txsize_to_bsize[l_max_tx_size] == bsizec);
if (is_split) max_tx_size = sub_tx_size_map[max_tx_size];
if (is_split) max_tx_size = sub_tx_size_map[1][max_tx_size];
}
#endif // DISABLE_VARTX_FOR_CHROMA == 2
......
......@@ -2322,8 +2322,7 @@ static int tx_size_cost(const AV1_COMMON *const cm, const MACROBLOCK *const x,
if (cm->tx_mode == TX_MODE_SELECT && block_signals_txsize(mbmi->sb_type)) {
const int is_inter = is_inter_block(mbmi);
const int32_t tx_size_cat = is_inter ? inter_tx_size_cat_lookup[bsize]
: intra_tx_size_cat_lookup[bsize];
const int32_t tx_size_cat = bsize_to_tx_size_cat(bsize, is_inter);
const int depth = tx_size_to_depth(tx_size, bsize, is_inter);
const int tx_size_ctx = get_tx_size_context(xd);
int r_tx_size = x->tx_size_cost[tx_size_cat][tx_size_ctx][depth];
......@@ -2617,7 +2616,8 @@ static void choose_tx_size_type_from_rd(const AV1_COMP *const cpi,
}
last_rd = INT64_MAX;
for (n = start_tx; depth <= MAX_TX_DEPTH; depth++, n = sub_tx_size_map[n]) {
for (n = start_tx; depth <= MAX_TX_DEPTH;
depth++, n = sub_tx_size_map[0][n]) {
TX_TYPE tx_start = DCT_DCT;
TX_TYPE tx_end = TX_TYPES;
#if CONFIG_TXK_SEL
......@@ -3938,7 +3938,7 @@ static void select_tx_block(const AV1_COMP *cpi, MACROBLOCK *x, int blk_row,
}
if (tx_size > TX_4X4 && depth < MAX_VARTX_DEPTH && tx_split_prune_flag == 0) {
const TX_SIZE sub_txs = sub_tx_size_map[tx_size];
const TX_SIZE sub_txs = sub_tx_size_map[1][tx_size];
const int bsw = tx_size_wide_unit[sub_txs];
const int bsh = tx_size_high_unit[sub_txs];
int sub_step = bsw * bsh;
......@@ -4360,7 +4360,7 @@ static void tx_block_yrd(const AV1_COMP *cpi, MACROBLOCK *x, int blk_row,
txfm_partition_update(tx_above + blk_col, tx_left + blk_row, tx_size,
tx_size);
} else {
const TX_SIZE sub_txs = sub_tx_size_map[tx_size];
const TX_SIZE sub_txs = sub_tx_size_map[1][tx_size];
const int bsw = tx_size_wide_unit[sub_txs];
const int bsh = tx_size_high_unit[sub_txs];
const int step = bsh * bsw;
......@@ -4587,7 +4587,7 @@ static int find_tx_size_rd_records(MACROBLOCK *x, BLOCK_SIZE bsize, int mi_row,
int cur_tx_depth = 0;
uint8_t parent_idx_buf[MAX_SB_SQUARE] = { 0 };
int cur_tx_size = max_txsize_rect_lookup[bsize];
int cur_tx_size = max_txsize_rect_lookup[1][bsize];
while (cur_tx_depth <= MAX_VARTX_DEPTH) {
const BLOCK_SIZE cur_tx_bsize = txsize_to_bsize[cur_tx_size];
const int cur_tx_bw = block_size_wide[cur_tx_bsize];
......@@ -4641,7 +4641,7 @@ static int find_tx_size_rd_records(MACROBLOCK *x, BLOCK_SIZE bsize, int mi_row,
++cur_rd_info_idx;
}
}
cur_tx_size = sub_tx_size_map[cur_tx_size];
cur_tx_size = sub_tx_size_map[1][cur_tx_size];
++cur_tx_depth;
}
return 1;
......@@ -4823,7 +4823,7 @@ static void select_tx_type_yrd(const AV1_COMP *cpi, MACROBLOCK *x,
TX_TYPE tx_type, best_tx_type = DCT_DCT;
const int is_inter = is_inter_block(mbmi);
TX_SIZE best_tx_size[MAX_MIB_SIZE][MAX_MIB_SIZE];
TX_SIZE best_tx = max_txsize_rect_lookup[bsize];
TX_SIZE best_tx = max_txsize_rect_lookup[1][bsize];
TX_SIZE best_min_tx_size = TX_SIZES_ALL;
uint8_t best_blk_skip[MAX_MIB_SIZE * MAX_MIB_SIZE * 8];
TX_TYPE txk_start = DCT_DCT;
......@@ -4836,7 +4836,7 @@ static void select_tx_type_yrd(const AV1_COMP *cpi, MACROBLOCK *x,
int idx, idy;
int prune = 0;
// Get the tx_size 1 level down
TX_SIZE min_tx_size = sub_tx_size_map[max_txsize_rect_lookup[bsize]];
TX_SIZE min_tx_size = sub_tx_size_map[1][max_txsize_rect_lookup[1][bsize]];
const TxSetType tx_set_type = get_ext_tx_set_type(
min_tx_size, bsize, is_inter, cm->reduced_tx_set_used);
int within_border = mi_row >= xd->tile.mi_row_start &&
......@@ -5001,7 +5001,7 @@ static void tx_block_rd(const AV1_COMP *cpi, MACROBLOCK *x, int blk_row,
plane_bsize, ta, tl, rd_stats, fast, NULL);
av1_set_txb_context(x, plane, block, tx_size, ta, tl);
} else {
const TX_SIZE sub_txs = sub_tx_size_map[tx_size];
const TX_SIZE sub_txs = sub_tx_size_map[1][tx_size];
assert(IMPLIES(tx_size <= TX_4X4, sub_txs == tx_size));
assert(IMPLIES(tx_size > TX_4X4, sub_txs < tx_size));
const int bsw = tx_size_wide_unit[sub_txs];
......@@ -5062,7 +5062,7 @@ int inter_block_uvrd(const AV1_COMP *cpi, MACROBLOCK *x, RD_STATS *rd_stats,
const int is_split =
(l_max_tx_size != mbmi->inter_tx_size[0][0] && bsize == bsizec &&
txsize_to_bsize[l_max_tx_size] == bsizec);
if (is_split) max_tx_size = sub_tx_size_map[max_tx_size];
if (is_split) max_tx_size = sub_tx_size_map[1][max_tx_size];
}
#endif // DISABLE_VARTX_FOR_CHROMA == 2
const int bh = tx_size_high_unit[max_tx_size];
......@@ -10619,7 +10619,7 @@ PALETTE_EXIT:
// Set up tx_size related variables for skip-specific loop filtering.
best_mbmode.tx_size = block_signals_txsize(bsize)
? tx_size_from_tx_mode(bsize, cm->tx_mode, 1)
: max_txsize_rect_lookup[bsize];
: max_txsize_rect_lookup[1][bsize];
{
const int width = block_size_wide[bsize] >> tx_size_wide_log2[0];
const int height = block_size_high[bsize] >> tx_size_high_log2[0];
......
......@@ -559,7 +559,7 @@ void tokenize_vartx(ThreadData *td, TOKENEXTRA **t, RUN_TYPE dry_run,
#endif
} else {
// Half the block size in transform block unit.
const TX_SIZE sub_txs = sub_tx_size_map[tx_size];
const TX_SIZE sub_txs = sub_tx_size_map[1][tx_size];
const int bsw = tx_size_wide_unit[sub_txs];
const int bsh = tx_size_high_unit[sub_txs];
const int step = bsw * bsh;
......@@ -640,7 +640,7 @@ void av1_tokenize_sb_vartx(const AV1_COMP *cpi, ThreadData *td, TOKENEXTRA **t,
const int is_split =
(l_max_tx_size != mbmi->inter_tx_size[0][0] && bsize == bsizec &&
txsize_to_bsize[l_max_tx_size] == bsizec);
if (is_split) max_tx_size = sub_tx_size_map[max_tx_size];
if (is_split) max_tx_size = sub_tx_size_map[1][max_tx_size];
}
#endif // DISABLE_VARTX_FOR_CHROMA == 2
const BLOCK_SIZE txb_size = txsize_to_bsize[max_tx_size];
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment