Commit c00769a9 authored by Yushin Cho's avatar Yushin Cho

[dist-8x8] Enable early-exit, if MSE is used

If dist-8x8 is enabled, it has disabled early-exit during the RD
of sub8x8 tx block in 8x8 or larger partition for plane 0.

However, for the sanity check mode of dist-8x8 which still use MSE,
this can cause different bdrate from the reference (i.e
dist-8x8 off). Hence, if tune-metirc is MSE then
regular early-exit will perform for all cases.

Note that the rd cost used for early-exit condition during tx mode
decision does not include rate for mode itself but only coefficints.
So, when the total distortion for a partition is very small, the rd cost
of mode can exceed that of distortion.

Change-Id: Ie25c64064b78d2d1582dbb0c286212fbc52dbdb9
parent bdddf33a
......@@ -147,7 +147,7 @@ static const int filter_sets[DUAL_FILTER_SET_SIZE][2] = {
// transform search. Trellis optimization will still be applied
// in the final encode.
#ifndef DISABLE_TRELLISQ_SEARCH
#define DISABLE_TRELLISQ_SEARCH 0
#define DISABLE_TRELLISQ_SEARCH 1
#endif
static const double ADST_FLIP_SVM[8] = {
......@@ -1984,10 +1984,18 @@ static void block_rd_txfm(int plane, int block, int blk_row, int blk_col,
#endif
int64_t rd1, rd2, rd;
RD_STATS this_rd_stats;
#if CONFIG_DIST_8X8 || !DISABLE_TRELLISQ_SEARCH
int sub8x8tx_in_gte8x8blk_in_plane0 =
plane == 0 && plane_bsize >= BLOCK_8X8 &&
(tx_size == TX_4X4 || tx_size == TX_4X8 || tx_size == TX_8X4);
#if CONFIG_DIST_8X8
// If sub8x8 tx, 8x8 or larger partition, and luma channel,
// dist-8x8 disables early skip, because the distortion metrics for
// sub8x8 tx (MSE) and reference distortion from 8x8 or larger partition
// (new distortion metric) are different.
// Exception is: dist-8x8 is enabled but still MSE is used,
// i.e. "--tune=" encoder option is not used.
int disable_early_skip =
x->using_dist_8x8 && plane == 0 && plane_bsize >= BLOCK_8X8 &&
(tx_size == TX_4X4 || tx_size == TX_4X8 || tx_size == TX_8X4) &&
x->tune_metric != AOM_TUNE_PSNR;
#endif // CONFIG_DIST_8X8
#if !CONFIG_SUPERTX && !CONFIG_VAR_TX
......@@ -2030,7 +2038,7 @@ static void block_rd_txfm(int plane, int block, int blk_row, int blk_col,
if (
#if CONFIG_DIST_8X8
(x->using_dist_8x8 && sub8x8tx_in_gte8x8blk_in_plane0) ||
disable_early_skip ||
#endif
RDCOST(x->rdmult, 0, tmp_dist) + args->this_rd < args->best_rd) {
av1_optimize_b(cm, x, plane, blk_row, blk_col, block, plane_bsize, tx_size,
......@@ -2117,15 +2125,12 @@ static void block_rd_txfm(int plane, int block, int blk_row, int blk_col,
args->this_rd += rd;
#if CONFIG_DIST_8X8
if (!x->using_dist_8x8 || !sub8x8tx_in_gte8x8blk_in_plane0) {
if (!disable_early_skip)
#endif
if (args->this_rd > args->best_rd) {
args->exit_early = 1;
return;
}
#if CONFIG_DIST_8X8
}
#endif
}
#if CONFIG_DIST_8X8
......@@ -4496,9 +4501,10 @@ void av1_tx_block_rd_b(const AV1_COMP *cpi, MACROBLOCK *x, TX_SIZE tx_size,
const int buffer_length = tx_size_2d[tx_size];
int64_t tmp_dist, tmp_sse;
#if CONFIG_DIST_8X8
int sub8x8tx_in_gte8x8blk_in_plane0 =
plane == 0 && plane_bsize >= BLOCK_8X8 &&
(tx_size == TX_4X4 || tx_size == TX_4X8 || tx_size == TX_8X4);
int disable_early_skip =
x->using_dist_8x8 && plane == 0 && plane_bsize >= BLOCK_8X8 &&
(tx_size == TX_4X4 || tx_size == TX_4X8 || tx_size == TX_8X4) &&
x->tune_metric != AOM_TUNE_PSNR;
#endif // CONFIG_DIST_8X8
#if CONFIG_HIGHBITDEPTH
......@@ -4519,7 +4525,7 @@ void av1_tx_block_rd_b(const AV1_COMP *cpi, MACROBLOCK *x, TX_SIZE tx_size,
#endif // CONFIG_MRC_TX
if (
#if CONFIG_DIST_8X8
(x->using_dist_8x8 && sub8x8tx_in_gte8x8blk_in_plane0) ||
disable_early_skip ||
#endif
RDCOST(x->rdmult, 0, tmp_dist) < rd_stats->ref_rdcost) {
av1_optimize_b(cm, x, plane, blk_row, blk_col, block, plane_bsize, tx_size,
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment