Commit 0fa057f8 authored by Debargha Mukherjee's avatar Debargha Mukherjee

Misc refactors to support 4:1->2:1->1:1 tx splits

Currently 4:1 transforms have max 2 split levels:
4:1 -> 1:1 -> 0.5:0.5.

This refactor enables split levels:
4:1 -> 2:1 -> 1:1,

by simply changing the tables in common_data.h.

The actual switch will be made in a subsequent patch.

Change-Id: I33f8d9ca5159ba3e7d02ced449ddf6f804a8f12a
parent 37131cfd
......@@ -1105,23 +1105,34 @@ static INLINE TX_TYPE av1_get_tx_type(PLANE_TYPE plane_type,
void av1_setup_block_planes(MACROBLOCKD *xd, int ss_x, int ss_y);
static INLINE int bsize_to_max_depth(BLOCK_SIZE bsize) {
const int32_t tx_size_cat = intra_tx_size_cat_lookup[bsize];
return AOMMIN(tx_size_cat + 1, MAX_TX_DEPTH);
static INLINE int bsize_to_max_depth(BLOCK_SIZE bsize, int is_inter) {
TX_SIZE tx_size = get_max_rect_tx_size(bsize, is_inter);
int depth = 0;
while (depth < MAX_TX_DEPTH && tx_size != TX_4X4) {
depth++;
tx_size = sub_tx_size_map[tx_size];
}
return depth;
}
static INLINE int tx_size_to_depth(TX_SIZE tx_size, BLOCK_SIZE bsize) {
if (tx_size == max_txsize_rect_intra_lookup[bsize]) return 0;
const int32_t tx_size_cat = intra_tx_size_cat_lookup[bsize];
const TX_SIZE coded_tx_size = txsize_sqr_map[tx_size];
return (int)(tx_size_cat + 1 - (int)coded_tx_size);
static INLINE int tx_size_to_depth(TX_SIZE tx_size, BLOCK_SIZE bsize,
int is_inter) {
TX_SIZE ctx_size = get_max_rect_tx_size(bsize, is_inter);
int depth = 0;
while (tx_size != ctx_size) {
depth++;
ctx_size = sub_tx_size_map[ctx_size];
assert(depth <= MAX_TX_DEPTH);
}
return depth;
}
static INLINE TX_SIZE depth_to_tx_size(int depth, BLOCK_SIZE bsize) {
if (depth == 0) return max_txsize_rect_intra_lookup[bsize];
const int32_t tx_size_cat = intra_tx_size_cat_lookup[bsize];
assert(tx_size_cat + 1 - depth >= 0 && tx_size_cat + 1 - depth < TX_SIZES);
return (TX_SIZE)(tx_size_cat + 1 - depth);
static INLINE TX_SIZE depth_to_tx_size(int depth, BLOCK_SIZE bsize,
int is_inter) {
TX_SIZE max_tx_size = get_max_rect_tx_size(bsize, is_inter);
TX_SIZE tx_size = max_tx_size;
for (int d = 0; d < depth; ++d) tx_size = sub_tx_size_map[tx_size];
return tx_size;
}
static INLINE TX_SIZE av1_get_uv_tx_size(const MB_MODE_INFO *mbmi,
......
......@@ -2815,7 +2815,6 @@ void av1_predict_intra_block(const AV1_COMMON *cm, const MACROBLOCKD *xd,
// A block should only fail to have a matching transform if it's
// large and rectangular (such large transform sizes aren't
// available).
assert(block_width >= 32 && block_height >= 32);
assert((block_width == wpx && block_height == hpx) ||
(block_width == (wpx >> 1) && block_height == hpx) ||
(block_width == wpx && block_height == (hpx >> 1)));
......
......@@ -504,7 +504,7 @@ static TX_SIZE read_selected_tx_size(AV1_COMMON *cm, MACROBLOCKD *xd,
const BLOCK_SIZE bsize = xd->mi[0]->mbmi.sb_type;
const int32_t tx_size_cat = is_inter ? inter_tx_size_cat_lookup[bsize]
: intra_tx_size_cat_lookup[bsize];
const int max_depths = bsize_to_max_depth(bsize);
const int max_depths = bsize_to_max_depth(bsize, 0);
const int ctx = get_tx_size_context(xd);
FRAME_CONTEXT *ec_ctx = xd->tile_ctx;
(void)cm;
......@@ -512,7 +512,7 @@ static TX_SIZE read_selected_tx_size(AV1_COMMON *cm, MACROBLOCKD *xd,
const int depth = aom_read_symbol(r, ec_ctx->tx_size_cdf[tx_size_cat][ctx],
max_depths + 1, ACCT_STR);
assert(depth >= 0 && depth <= max_depths);
const TX_SIZE tx_size = depth_to_tx_size(depth, bsize);
const TX_SIZE tx_size = depth_to_tx_size(depth, bsize, 0);
return tx_size;
}
......
......@@ -263,8 +263,8 @@ static void write_selected_tx_size(const AV1_COMMON *cm, const MACROBLOCKD *xd,
const TX_SIZE tx_size = mbmi->tx_size;
const int tx_size_ctx = get_tx_size_context(xd);
const int32_t tx_size_cat = intra_tx_size_cat_lookup[bsize];
const int depth = tx_size_to_depth(tx_size, bsize);
const int max_depths = bsize_to_max_depth(bsize);
const int depth = tx_size_to_depth(tx_size, bsize, 0);
const int max_depths = bsize_to_max_depth(bsize, 0);
assert(depth >= 0 && depth <= max_depths);
assert(!is_inter_block(mbmi));
......
......@@ -429,11 +429,8 @@ static void reset_tx_size(MACROBLOCKD *xd, MB_MODE_INFO *mbmi,
tx_size_from_tx_mode(mbmi->sb_type, tx_mode, is_inter_block(mbmi));
} else {
BLOCK_SIZE bsize = mbmi->sb_type;
TX_SIZE max_rect_txsize = get_max_rect_tx_size(bsize, is_inter_block(mbmi));
TX_SIZE min_tx_size =
(TX_SIZE)AOMMAX((int)TX_4X4,
txsize_sqr_map[max_rect_txsize] - MAX_TX_DEPTH +
is_rect_tx(max_rect_txsize));
depth_to_tx_size(MAX_TX_DEPTH, bsize, is_inter_block(mbmi));
mbmi->tx_size = (TX_SIZE)AOMMAX(mbmi->tx_size, min_tx_size);
}
}
......@@ -4538,8 +4535,8 @@ static void sum_intra_stats(FRAME_COUNTS *counts, MACROBLOCKD *xd,
const TX_SIZE tx_size = mbmi->tx_size;
const int tx_size_ctx = get_tx_size_context(xd);
const int32_t tx_size_cat = intra_tx_size_cat_lookup[bsize];
const int depth = tx_size_to_depth(tx_size, bsize);
const int max_depths = bsize_to_max_depth(bsize);
const int depth = tx_size_to_depth(tx_size, bsize, 0);
const int max_depths = bsize_to_max_depth(bsize, 0);
update_cdf(fc->tx_size_cdf[tx_size_cat][tx_size_ctx], depth,
max_depths + 1);
}
......
......@@ -2324,7 +2324,7 @@ static int tx_size_cost(const AV1_COMMON *const cm, const MACROBLOCK *const x,
const int is_inter = is_inter_block(mbmi);
const int32_t tx_size_cat = is_inter ? inter_tx_size_cat_lookup[bsize]
: intra_tx_size_cat_lookup[bsize];
const int depth = tx_size_to_depth(tx_size, bsize);
const int depth = tx_size_to_depth(tx_size, bsize, is_inter);
const int tx_size_ctx = get_tx_size_context(xd);
int r_tx_size = x->tx_size_cost[tx_size_cat][tx_size_ctx][depth];
return r_tx_size;
......@@ -2437,7 +2437,6 @@ static int skip_txfm_search(const AV1_COMP *cpi, MACROBLOCK *x, BLOCK_SIZE bs,
TX_TYPE tx_type, TX_SIZE tx_size, int prune) {
const MACROBLOCKD *const xd = &x->e_mbd;
const MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi;
const TX_SIZE max_tx_size = max_txsize_lookup[bs];
const int is_inter = is_inter_block(mbmi);
if (mbmi->ref_mv_idx > 0 && tx_type != DCT_DCT) return 1;
......@@ -2449,7 +2448,6 @@ static int skip_txfm_search(const AV1_COMP *cpi, MACROBLOCK *x, BLOCK_SIZE bs,
if (is_inter && x->use_default_inter_tx_type &&
tx_type != get_default_tx_type(0, xd, 0, tx_size))
return 1;
if (max_tx_size >= TX_32X32 && tx_size == TX_4X4) return 1;
const AV1_COMMON *const cm = &cpi->common;
const TxSetType tx_set_type =
get_ext_tx_set_type(tx_size, bs, is_inter, cm->reduced_tx_set_used);
......@@ -2588,74 +2586,28 @@ static void choose_tx_size_type_from_rd(const AV1_COMP *const cpi,
MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi;
int64_t rd = INT64_MAX;
int n;
int start_tx, end_tx;
int start_tx;
int depth;
int64_t best_rd = INT64_MAX, last_rd = INT64_MAX;
const TX_SIZE max_tx_size = max_txsize_lookup[bs];
TX_SIZE best_tx_size = max_tx_size;
const int is_inter = is_inter_block(mbmi);
const TX_SIZE max_rect_tx_size = get_max_rect_tx_size(bs, is_inter);
TX_SIZE best_tx_size = max_rect_tx_size;
TX_TYPE best_tx_type = DCT_DCT;
#if CONFIG_TXK_SEL
TX_TYPE best_txk_type[MAX_SB_SQUARE / (TX_SIZE_W_MIN * TX_SIZE_H_MIN)];
#endif // CONFIG_TXK_SEL
const int tx_select = cm->tx_mode == TX_MODE_SELECT;
const int is_inter = is_inter_block(mbmi);
av1_invalid_rd_stats(rd_stats);
int evaluate_rect_tx = 0;
if (tx_select) {
evaluate_rect_tx = is_rect_tx_allowed(xd, mbmi);
} else {
const TX_SIZE chosen_tx_size =
tx_size_from_tx_mode(bs, cm->tx_mode, is_inter);
evaluate_rect_tx = is_rect_tx(chosen_tx_size);
assert(IMPLIES(evaluate_rect_tx, is_rect_tx_allowed(xd, mbmi)));
}
if (evaluate_rect_tx) {
TX_TYPE tx_start = DCT_DCT;
TX_TYPE tx_end = TX_TYPES;
#if CONFIG_TXK_SEL
// The tx_type becomes dummy when lv_map is on. The tx_type search will be
// performed in av1_search_txk_type()
tx_end = DCT_DCT + 1;
#endif
TX_TYPE tx_type;
for (tx_type = tx_start; tx_type < tx_end; ++tx_type) {
if (mbmi->ref_mv_idx > 0 && tx_type != DCT_DCT) continue;
const TX_SIZE rect_tx_size = get_max_rect_tx_size(bs, is_inter);
RD_STATS this_rd_stats;
const TxSetType tx_set_type = get_ext_tx_set_type(
rect_tx_size, bs, is_inter, cm->reduced_tx_set_used);
if (av1_ext_tx_used[tx_set_type][tx_type]) {
rd = txfm_yrd(cpi, x, &this_rd_stats, ref_best_rd, bs, tx_type,
rect_tx_size);
ref_best_rd = AOMMIN(rd, ref_best_rd);
if (rd < best_rd) {
#if CONFIG_TXK_SEL
memcpy(best_txk_type, mbmi->txk_type,
sizeof(best_txk_type[0]) * MAX_SB_SQUARE /
(TX_SIZE_W_MIN * TX_SIZE_H_MIN));
#endif
best_tx_type = tx_type;
best_tx_size = rect_tx_size;
best_rd = rd;
*rd_stats = this_rd_stats;
}
}
#if !USE_TXTYPE_SEARCH_FOR_SUB8X8_IN_CB4X4
const int is_inter = is_inter_block(mbmi);
if (mbmi->sb_type < BLOCK_8X8 && is_inter) break;
#endif // !USE_TXTYPE_SEARCH_FOR_SUB8X8_IN_CB4X4
}
}
if (tx_select) {
start_tx = max_tx_size;
end_tx = AOMMAX((int)TX_4X4, start_tx - MAX_TX_DEPTH + evaluate_rect_tx);
start_tx = max_rect_tx_size;
depth = 0;
} else {
const TX_SIZE chosen_tx_size =
tx_size_from_tx_mode(bs, cm->tx_mode, is_inter);
start_tx = chosen_tx_size;
end_tx = chosen_tx_size;
depth = MAX_TX_DEPTH;
}
int prune = 0;
......@@ -2665,8 +2617,7 @@ static void choose_tx_size_type_from_rd(const AV1_COMP *const cpi,
}
last_rd = INT64_MAX;
for (n = start_tx; n >= end_tx; --n) {
if (is_rect_tx(n)) break;
for (n = start_tx; depth <= MAX_TX_DEPTH; depth++, n = sub_tx_size_map[n]) {
TX_TYPE tx_start = DCT_DCT;
TX_TYPE tx_end = TX_TYPES;
#if CONFIG_TXK_SEL
......@@ -2683,8 +2634,8 @@ static void choose_tx_size_type_from_rd(const AV1_COMP *const cpi,
// Early termination in transform size search.
if (cpi->sf.tx_size_search_breakout &&
(rd == INT64_MAX ||
(this_rd_stats.skip == 1 && tx_type != DCT_DCT && n < start_tx) ||
(n < (int)max_tx_size && rd > last_rd))) {
(this_rd_stats.skip == 1 && tx_type != DCT_DCT && n != start_tx) ||
(n != (int)start_tx && rd > last_rd))) {
break;
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment