Commit 421af353 authored by Jingning Han's avatar Jingning Han Committed by Gerrit Code Review

Merge "Limit the transform block partition depth" into nextgenv2

parents bb22bbf0 94d5bfcc
...@@ -276,8 +276,8 @@ static int read_segment_id(aom_reader *r, ...@@ -276,8 +276,8 @@ static int read_segment_id(aom_reader *r,
#if CONFIG_VAR_TX #if CONFIG_VAR_TX
static void read_tx_size_vartx(AV1_COMMON *cm, MACROBLOCKD *xd, static void read_tx_size_vartx(AV1_COMMON *cm, MACROBLOCKD *xd,
MB_MODE_INFO *mbmi, FRAME_COUNTS *counts, MB_MODE_INFO *mbmi, FRAME_COUNTS *counts,
TX_SIZE tx_size, int blk_row, int blk_col, TX_SIZE tx_size, int depth, int blk_row,
aom_reader *r) { int blk_col, aom_reader *r) {
int is_split = 0; int is_split = 0;
const int tx_row = blk_row >> 1; const int tx_row = blk_row >> 1;
const int tx_col = blk_col >> 1; const int tx_col = blk_col >> 1;
...@@ -294,6 +294,19 @@ static void read_tx_size_vartx(AV1_COMMON *cm, MACROBLOCKD *xd, ...@@ -294,6 +294,19 @@ static void read_tx_size_vartx(AV1_COMMON *cm, MACROBLOCKD *xd,
if (blk_row >= max_blocks_high || blk_col >= max_blocks_wide) return; if (blk_row >= max_blocks_high || blk_col >= max_blocks_wide) return;
if (depth == 2) {
int idx, idy;
inter_tx_size[0][0] = tx_size;
for (idy = 0; idy < num_4x4_blocks_high_txsize_lookup[tx_size] / 2; ++idy)
for (idx = 0; idx < num_4x4_blocks_wide_txsize_lookup[tx_size] / 2; ++idx)
inter_tx_size[idy][idx] = tx_size;
mbmi->tx_size = tx_size;
if (counts) ++counts->txfm_partition[ctx][0];
txfm_partition_update(xd->above_txfm_context + tx_col,
xd->left_txfm_context + tx_row, tx_size);
return;
}
is_split = aom_read(r, cm->fc->txfm_partition_prob[ctx], ACCT_STR); is_split = aom_read(r, cm->fc->txfm_partition_prob[ctx], ACCT_STR);
if (is_split) { if (is_split) {
...@@ -316,8 +329,8 @@ static void read_tx_size_vartx(AV1_COMMON *cm, MACROBLOCKD *xd, ...@@ -316,8 +329,8 @@ static void read_tx_size_vartx(AV1_COMMON *cm, MACROBLOCKD *xd,
for (i = 0; i < 4; ++i) { for (i = 0; i < 4; ++i) {
int offsetr = blk_row + ((i >> 1) << bsl); int offsetr = blk_row + ((i >> 1) << bsl);
int offsetc = blk_col + ((i & 0x01) << bsl); int offsetc = blk_col + ((i & 0x01) << bsl);
read_tx_size_vartx(cm, xd, mbmi, counts, tx_size - 1, offsetr, offsetc, read_tx_size_vartx(cm, xd, mbmi, counts, tx_size - 1, depth + 1, offsetr,
r); offsetc, r);
} }
} else { } else {
int idx, idy; int idx, idy;
...@@ -1811,8 +1824,8 @@ static void read_inter_frame_mode_info(AV1Decoder *const pbi, ...@@ -1811,8 +1824,8 @@ static void read_inter_frame_mode_info(AV1Decoder *const pbi,
#endif // CONFIG_EXT_TX && CONFIG_RECT_TX #endif // CONFIG_EXT_TX && CONFIG_RECT_TX
for (idy = 0; idy < height; idy += bs) for (idy = 0; idy < height; idy += bs)
for (idx = 0; idx < width; idx += bs) for (idx = 0; idx < width; idx += bs)
read_tx_size_vartx(cm, xd, mbmi, xd->counts, max_tx_size, idy, idx, read_tx_size_vartx(cm, xd, mbmi, xd->counts, max_tx_size,
r); height != width, idy, idx, r);
#if CONFIG_EXT_TX && CONFIG_RECT_TX #if CONFIG_EXT_TX && CONFIG_RECT_TX
} }
#endif #endif
......
...@@ -365,7 +365,8 @@ static int prob_diff_update_savings(const aom_tree_index *tree, ...@@ -365,7 +365,8 @@ static int prob_diff_update_savings(const aom_tree_index *tree,
#if CONFIG_VAR_TX #if CONFIG_VAR_TX
static void write_tx_size_vartx(const AV1_COMMON *cm, const MACROBLOCKD *xd, static void write_tx_size_vartx(const AV1_COMMON *cm, const MACROBLOCKD *xd,
const MB_MODE_INFO *mbmi, TX_SIZE tx_size, const MB_MODE_INFO *mbmi, TX_SIZE tx_size,
int blk_row, int blk_col, aom_writer *w) { int depth, int blk_row, int blk_col,
aom_writer *w) {
const int tx_row = blk_row >> 1; const int tx_row = blk_row >> 1;
const int tx_col = blk_col >> 1; const int tx_col = blk_col >> 1;
int max_blocks_high = num_4x4_blocks_high_lookup[mbmi->sb_type]; int max_blocks_high = num_4x4_blocks_high_lookup[mbmi->sb_type];
...@@ -378,6 +379,12 @@ static void write_tx_size_vartx(const AV1_COMMON *cm, const MACROBLOCKD *xd, ...@@ -378,6 +379,12 @@ static void write_tx_size_vartx(const AV1_COMMON *cm, const MACROBLOCKD *xd,
if (blk_row >= max_blocks_high || blk_col >= max_blocks_wide) return; if (blk_row >= max_blocks_high || blk_col >= max_blocks_wide) return;
if (depth == 2) {
txfm_partition_update(xd->above_txfm_context + tx_col,
xd->left_txfm_context + tx_row, tx_size);
return;
}
if (tx_size == mbmi->inter_tx_size[tx_row][tx_col]) { if (tx_size == mbmi->inter_tx_size[tx_row][tx_col]) {
aom_write(w, 0, cm->fc->txfm_partition_prob[ctx]); aom_write(w, 0, cm->fc->txfm_partition_prob[ctx]);
txfm_partition_update(xd->above_txfm_context + tx_col, txfm_partition_update(xd->above_txfm_context + tx_col,
...@@ -399,7 +406,8 @@ static void write_tx_size_vartx(const AV1_COMMON *cm, const MACROBLOCKD *xd, ...@@ -399,7 +406,8 @@ static void write_tx_size_vartx(const AV1_COMMON *cm, const MACROBLOCKD *xd,
for (i = 0; i < 4; ++i) { for (i = 0; i < 4; ++i) {
int offsetr = blk_row + ((i >> 1) << bsl); int offsetr = blk_row + ((i >> 1) << bsl);
int offsetc = blk_col + ((i & 0x01) << bsl); int offsetc = blk_col + ((i & 0x01) << bsl);
write_tx_size_vartx(cm, xd, mbmi, tx_size - 1, offsetr, offsetc, w); write_tx_size_vartx(cm, xd, mbmi, tx_size - 1, depth + 1, offsetr,
offsetc, w);
} }
} }
} }
...@@ -1229,7 +1237,8 @@ static void pack_inter_mode_mvs(AV1_COMP *cpi, const MODE_INFO *mi, ...@@ -1229,7 +1237,8 @@ static void pack_inter_mode_mvs(AV1_COMP *cpi, const MODE_INFO *mi,
#endif // CONFIG_EXT_TX && CONFIG_RECT_TX #endif // CONFIG_EXT_TX && CONFIG_RECT_TX
for (idy = 0; idy < height; idy += bs) for (idy = 0; idy < height; idy += bs)
for (idx = 0; idx < width; idx += bs) for (idx = 0; idx < width; idx += bs)
write_tx_size_vartx(cm, xd, mbmi, max_tx_size, idy, idx, w); write_tx_size_vartx(cm, xd, mbmi, max_tx_size, height != width, idy,
idx, w);
#if CONFIG_EXT_TX && CONFIG_RECT_TX #if CONFIG_EXT_TX && CONFIG_RECT_TX
} }
#endif // CONFIG_EXT_TX && CONFIG_RECT_TX #endif // CONFIG_EXT_TX && CONFIG_RECT_TX
......
...@@ -3020,11 +3020,11 @@ void av1_tx_block_rd_b(const AV1_COMP *cpi, MACROBLOCK *x, TX_SIZE tx_size, ...@@ -3020,11 +3020,11 @@ void av1_tx_block_rd_b(const AV1_COMP *cpi, MACROBLOCK *x, TX_SIZE tx_size,
static void select_tx_block(const AV1_COMP *cpi, MACROBLOCK *x, int blk_row, static void select_tx_block(const AV1_COMP *cpi, MACROBLOCK *x, int blk_row,
int blk_col, int plane, int block, TX_SIZE tx_size, int blk_col, int plane, int block, TX_SIZE tx_size,
BLOCK_SIZE plane_bsize, ENTROPY_CONTEXT *ta, int depth, BLOCK_SIZE plane_bsize,
ENTROPY_CONTEXT *tl, TXFM_CONTEXT *tx_above, ENTROPY_CONTEXT *ta, ENTROPY_CONTEXT *tl,
TXFM_CONTEXT *tx_left, int *rate, int64_t *dist, TXFM_CONTEXT *tx_above, TXFM_CONTEXT *tx_left,
int64_t *bsse, int *skip, int64_t ref_best_rd, int *rate, int64_t *dist, int64_t *bsse, int *skip,
int *is_cost_valid) { int64_t ref_best_rd, int *is_cost_valid) {
MACROBLOCKD *const xd = &x->e_mbd; MACROBLOCKD *const xd = &x->e_mbd;
MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi; MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi;
struct macroblock_plane *const p = &x->plane[plane]; struct macroblock_plane *const p = &x->plane[plane];
...@@ -3116,13 +3116,13 @@ static void select_tx_block(const AV1_COMP *cpi, MACROBLOCK *x, int blk_row, ...@@ -3116,13 +3116,13 @@ static void select_tx_block(const AV1_COMP *cpi, MACROBLOCK *x, int blk_row,
*skip = 0; *skip = 0;
} }
if (tx_size > TX_4X4) if (tx_size > TX_4X4 && depth < 2)
*rate += av1_cost_bit(cpi->common.fc->txfm_partition_prob[ctx], 0); *rate += av1_cost_bit(cpi->common.fc->txfm_partition_prob[ctx], 0);
this_rd = RDCOST(x->rdmult, x->rddiv, *rate, *dist); this_rd = RDCOST(x->rdmult, x->rddiv, *rate, *dist);
tmp_eob = p->eobs[block]; tmp_eob = p->eobs[block];
} }
if (tx_size > TX_4X4) { if (tx_size > TX_4X4 && depth < 2) {
BLOCK_SIZE bsize = txsize_to_bsize[tx_size]; BLOCK_SIZE bsize = txsize_to_bsize[tx_size];
int bsl = b_height_log2_lookup[bsize]; int bsl = b_height_log2_lookup[bsize];
int sub_step = num_4x4_blocks_txsize_lookup[tx_size - 1]; int sub_step = num_4x4_blocks_txsize_lookup[tx_size - 1];
...@@ -3141,9 +3141,10 @@ static void select_tx_block(const AV1_COMP *cpi, MACROBLOCK *x, int blk_row, ...@@ -3141,9 +3141,10 @@ static void select_tx_block(const AV1_COMP *cpi, MACROBLOCK *x, int blk_row,
int offsetr = (i >> 1) << bsl; int offsetr = (i >> 1) << bsl;
int offsetc = (i & 0x01) << bsl; int offsetc = (i & 0x01) << bsl;
select_tx_block(cpi, x, blk_row + offsetr, blk_col + offsetc, plane, select_tx_block(cpi, x, blk_row + offsetr, blk_col + offsetc, plane,
block + i * sub_step, tx_size - 1, plane_bsize, ta, tl, block + i * sub_step, tx_size - 1, depth + 1, plane_bsize,
tx_above, tx_left, &this_rate, &this_dist, &this_bsse, ta, tl, tx_above, tx_left, &this_rate, &this_dist,
&this_skip, ref_best_rd - tmp_rd, &this_cost_valid); &this_bsse, &this_skip, ref_best_rd - tmp_rd,
&this_cost_valid);
sum_rate += this_rate; sum_rate += this_rate;
sum_dist += this_dist; sum_dist += this_dist;
sum_bsse += this_bsse; sum_bsse += this_bsse;
...@@ -3219,9 +3220,10 @@ static void inter_block_yrd(const AV1_COMP *cpi, MACROBLOCK *x, int *rate, ...@@ -3219,9 +3220,10 @@ static void inter_block_yrd(const AV1_COMP *cpi, MACROBLOCK *x, int *rate,
for (idy = 0; idy < mi_height; idy += bh) { for (idy = 0; idy < mi_height; idy += bh) {
for (idx = 0; idx < mi_width; idx += bh) { for (idx = 0; idx < mi_width; idx += bh) {
select_tx_block(cpi, x, idy, idx, 0, block, select_tx_block(cpi, x, idy, idx, 0, block,
max_txsize_lookup[plane_bsize], plane_bsize, ctxa, ctxl, max_txsize_lookup[plane_bsize], mi_height != mi_width,
tx_above, tx_left, &pnrate, &pndist, &pnsse, &pnskip, plane_bsize, ctxa, ctxl, tx_above, tx_left, &pnrate,
ref_best_rd - this_rd, &is_cost_valid); &pndist, &pnsse, &pnskip, ref_best_rd - this_rd,
&is_cost_valid);
*rate += pnrate; *rate += pnrate;
*distortion += pndist; *distortion += pndist;
*sse += pnsse; *sse += pnsse;
...@@ -7586,7 +7588,7 @@ static int64_t handle_inter_mode( ...@@ -7586,7 +7588,7 @@ static int64_t handle_inter_mode(
// Y cost and distortion // Y cost and distortion
av1_subtract_plane(x, bsize, 0); av1_subtract_plane(x, bsize, 0);
#if CONFIG_VAR_TX #if CONFIG_VAR_TX
if (cm->tx_mode == TX_MODE_SELECT || xd->lossless[mbmi->segment_id]) { if (cm->tx_mode == TX_MODE_SELECT && !xd->lossless[mbmi->segment_id]) {
select_tx_type_yrd(cpi, x, rate_y, &distortion_y, &skippable_y, psse, select_tx_type_yrd(cpi, x, rate_y, &distortion_y, &skippable_y, psse,
bsize, ref_best_rd); bsize, ref_best_rd);
} else { } else {
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment