Commit 3bce7547 authored by Jingning Han's avatar Jingning Han

Optimize transform block rate-distortion search

The soft coefficient optimization process would monotonically
increase the transform block distortion and decrease the
coefficient rate cost. Such observation provides a lower bound
on the rate-distortion cost for the given transform block. This
commit compares this lower bound against the best available
rate-distortion cost value and skips unnecessary optimization
process. It speeds up the baseline encoding process by 15%.

Change-Id: Ida8098a2820cef60d59ec1e72f0bbb1acbd98165
parent 53a04f66
......@@ -326,6 +326,8 @@ typedef struct RD_STATS {
int64_t rdcost;
int64_t sse;
int skip; // sse should equal to dist when skip == 1
int64_t ref_rdcost;
int zero_rate;
#if CONFIG_DIST_8X8 && CONFIG_CB4X4
int64_t dist_y;
#endif
......
......@@ -457,6 +457,8 @@ static INLINE void av1_init_rd_stats(RD_STATS *rd_stats) {
rd_stats->rdcost = 0;
rd_stats->sse = 0;
rd_stats->skip = 1;
rd_stats->zero_rate = 0;
rd_stats->ref_rdcost = INT64_MAX;
#if CONFIG_DIST_8X8 && CONFIG_CB4X4
rd_stats->dist_y = 0;
#endif
......@@ -484,6 +486,8 @@ static INLINE void av1_invalid_rd_stats(RD_STATS *rd_stats) {
rd_stats->rdcost = INT64_MAX;
rd_stats->sse = INT64_MAX;
rd_stats->skip = 0;
rd_stats->zero_rate = 0;
rd_stats->ref_rdcost = INT64_MAX;
#if CONFIG_DIST_8X8 && CONFIG_CB4X4
rd_stats->dist_y = INT64_MAX;
#endif
......
......@@ -1893,8 +1893,29 @@ static void block_rd_txfm(int plane, int block, int blk_row, int blk_col,
#else
av1_xform_quant(cm, x, plane, block, blk_row, blk_col, plane_bsize, tx_size,
coeff_ctx, AV1_XFORM_QUANT_FP);
av1_optimize_b(cm, x, plane, blk_row, blk_col, block, plane_bsize, tx_size, a,
l);
const int shift = (MAX_TX_SCALE - av1_get_tx_scale(tx_size)) * 2;
tran_low_t *const coeff = BLOCK_OFFSET(x->plane[plane].coeff, block);
tran_low_t *const dqcoeff = BLOCK_OFFSET(xd->plane[plane].dqcoeff, block);
const int buffer_length = tx_size_2d[tx_size];
int64_t tmp_dist;
int64_t tmp;
#if CONFIG_HIGHBITDEPTH
if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH)
tmp_dist =
av1_highbd_block_error(coeff, dqcoeff, buffer_length, &tmp, xd->bd) >>
shift;
else
#endif
tmp_dist = av1_block_error(coeff, dqcoeff, buffer_length, &tmp) >> shift;
if (RDCOST(x->rdmult, 0, tmp_dist) + args->this_rd < args->best_rd) {
av1_optimize_b(cm, x, plane, blk_row, blk_col, block, plane_bsize, tx_size,
a, l);
} else {
args->exit_early = 1;
return;
}
#endif // DISABLE_TRELLISQ_SEARCH
if (!is_inter_block(mbmi)) {
......@@ -4307,18 +4328,6 @@ void av1_tx_block_rd_b(const AV1_COMP *cpi, MACROBLOCK *x, TX_SIZE tx_size,
int coeff_ctx = get_entropy_context(tx_size, a, l);
#if DISABLE_TRELLISQ_SEARCH
av1_xform_quant(cm, x, plane, block, blk_row, blk_col, plane_bsize, tx_size,
coeff_ctx, AV1_XFORM_QUANT_B);
#else
av1_xform_quant(cm, x, plane, block, blk_row, blk_col, plane_bsize, tx_size,
coeff_ctx, AV1_XFORM_QUANT_FP);
av1_optimize_b(cm, x, plane, blk_row, blk_col, block, plane_bsize, tx_size, a,
l);
#endif // DISABLE_TRELLISQ_SEARCH
// TODO(any): Use av1_dist_block to compute distortion
#if CONFIG_HIGHBITDEPTH
if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) {
......@@ -4335,6 +4344,33 @@ void av1_tx_block_rd_b(const AV1_COMP *cpi, MACROBLOCK *x, TX_SIZE tx_size,
0, bw, bh);
#endif // CONFIG_HIGHBITDEPTH
#if DISABLE_TRELLISQ_SEARCH
av1_xform_quant(cm, x, plane, block, blk_row, blk_col, plane_bsize, tx_size,
coeff_ctx, AV1_XFORM_QUANT_B);
#else
av1_xform_quant(cm, x, plane, block, blk_row, blk_col, plane_bsize, tx_size,
coeff_ctx, AV1_XFORM_QUANT_FP);
const int shift = (MAX_TX_SCALE - av1_get_tx_scale(tx_size)) * 2;
tran_low_t *const coeff = BLOCK_OFFSET(p->coeff, block);
const int buffer_length = tx_size_2d[tx_size];
int64_t tmp_dist;
#if CONFIG_HIGHBITDEPTH
if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH)
tmp_dist =
av1_highbd_block_error(coeff, dqcoeff, buffer_length, &tmp, xd->bd) >>
shift;
else
#endif
tmp_dist = av1_block_error(coeff, dqcoeff, buffer_length, &tmp) >> shift;
if (RDCOST(x->rdmult, 0, tmp_dist) < rd_stats->ref_rdcost) {
av1_optimize_b(cm, x, plane, blk_row, blk_col, block, plane_bsize, tx_size,
a, l);
}
#endif // DISABLE_TRELLISQ_SEARCH
tmp = pixel_diff_dist(x, plane, diff, diff_stride, blk_row, blk_col,
plane_bsize, txm_bsize);
......@@ -4456,6 +4492,8 @@ static void select_tx_block(const AV1_COMP *cpi, MACROBLOCK *x, int blk_row,
[coeff_ctx][EOB_TOKEN];
#endif
rd_stats->ref_rdcost = ref_best_rd;
rd_stats->zero_rate = zero_blk_rate;
if (cpi->common.tx_mode == TX_MODE_SELECT || tx_size == TX_4X4) {
inter_tx_size[0][0] = tx_size;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment